file_update.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Load the JSON file into a DataFrame\n",
    "file_path = (\"company_tickers.json\"  \n",
    ")\n",
    "df = pd.read_json(file_path)\n",
    "\n",
    "df = df.T\n",
    "\n",
    "# Add a new column for processing status\n",
    "df[\"processed\"] = False\n",
    "\n",
    "# Save the DataFrame to a CSV file\n",
    "status_file_path = \"processing_status.csv\"  \n",
    "df.to_csv(status_file_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cik</th>\n",
       "      <th>ticker</th>\n",
       "      <th>title</th>\n",
       "      <th>year</th>\n",
       "      <th>mda_section</th>\n",
       "      <th>processed_timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>789019</td>\n",
       "      <td>MSFT</td>\n",
       "      <td>MICROSOFT CORP</td>\n",
       "      <td>2023</td>\n",
       "      <td>ITEM 7. MANAGEMENT’S DISCUSSION AND ANALYSIS O...</td>\n",
       "      <td>2024-01-14 21:34:52.582424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>789019</td>\n",
       "      <td>MSFT</td>\n",
       "      <td>MICROSOFT CORP</td>\n",
       "      <td>2002</td>\n",
       "      <td>ITEM 7. MANAGEMENT’S DISCUSSION AND ANALYSIS O...</td>\n",
       "      <td>2024-01-14 21:31:45.152628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>789019</td>\n",
       "      <td>MSFT</td>\n",
       "      <td>MICROSOFT CORP</td>\n",
       "      <td>2003</td>\n",
       "      <td>ITEM 7. Management’s Discussion and Analysis o...</td>\n",
       "      <td>2024-01-14 21:30:17.562345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>789019</td>\n",
       "      <td>MSFT</td>\n",
       "      <td>MICROSOFT CORP</td>\n",
       "      <td>2004</td>\n",
       "      <td>ITEM 7. MANAGEMENT’S DISCUSSION AND ANALYSIS O...</td>\n",
       "      <td>2024-01-14 21:31:02.722148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>789019</td>\n",
       "      <td>MSFT</td>\n",
       "      <td>MICROSOFT CORP</td>\n",
       "      <td>2005</td>\n",
       "      <td></td>\n",
       "      <td>2024-01-14 21:31:44.872061</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      cik ticker           title  year  \\\n",
       "0  789019   MSFT  MICROSOFT CORP  2023   \n",
       "1  789019   MSFT  MICROSOFT CORP  2002   \n",
       "2  789019   MSFT  MICROSOFT CORP  2003   \n",
       "3  789019   MSFT  MICROSOFT CORP  2004   \n",
       "4  789019   MSFT  MICROSOFT CORP  2005   \n",
       "\n",
       "                                         mda_section  \\\n",
       "0  ITEM 7. MANAGEMENT’S DISCUSSION AND ANALYSIS O...   \n",
       "1  ITEM 7. MANAGEMENT’S DISCUSSION AND ANALYSIS O...   \n",
       "2  ITEM 7. Management’s Discussion and Analysis o...   \n",
       "3  ITEM 7. MANAGEMENT’S DISCUSSION AND ANALYSIS O...   \n",
       "4                                                      \n",
       "\n",
       "         processed_timestamp  \n",
       "0 2024-01-14 21:34:52.582424  \n",
       "1 2024-01-14 21:31:45.152628  \n",
       "2 2024-01-14 21:30:17.562345  \n",
       "3 2024-01-14 21:31:02.722148  \n",
       "4 2024-01-14 21:31:44.872061  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd \n",
    "# read paraquet file\n",
    "df = pd.read_parquet('ticker_data/MSFT.parquet')\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cik_str</th>\n",
       "      <th>ticker</th>\n",
       "      <th>title</th>\n",
       "      <th>processed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>320193</td>\n",
       "      <td>AAPL</td>\n",
       "      <td>Apple Inc.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>789019</td>\n",
       "      <td>MSFT</td>\n",
       "      <td>MICROSOFT CORP</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1652044</td>\n",
       "      <td>GOOGL</td>\n",
       "      <td>Alphabet Inc.</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1018724</td>\n",
       "      <td>AMZN</td>\n",
       "      <td>AMAZON COM INC</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1045810</td>\n",
       "      <td>NVDA</td>\n",
       "      <td>NVIDIA CORP</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cik_str ticker           title  processed\n",
       "0   320193   AAPL      Apple Inc.      False\n",
       "1   789019   MSFT  MICROSOFT CORP      False\n",
       "2  1652044  GOOGL   Alphabet Inc.      False\n",
       "3  1018724   AMZN  AMAZON COM INC      False\n",
       "4  1045810   NVDA     NVIDIA CORP      False"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"processing_status.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "processed\n",
       "False    10909\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Total count of processed and unprocessed rows\n",
    "df[\"processed\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}