Script to extract data from spreadsheet and uploading it into firestore

script.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python391jvsc74a57bd0a9dc8b5fcedee32f1a73787c228213d69d063ef1ba8828f12ff43e4622e0bf8d",
   "display_name": "Python 3.9.1 64-bit"
  },
  "metadata": {
   "interpreter": {
    "hash": "a9dc8b5fcedee32f1a73787c228213d69d063ef1ba8828f12ff43e4622e0bf8d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Processed 6 lines.\n",
      "6\n",
      "<generator object batch_data at 0x000002606DF8A9E0>\n",
      "[{'row Number': '0', 'sno': '1', 'name': 'm'}, {'row Number': '1', 'sno': '2', 'name': 'o'}, {'row Number': '2', 'sno': '3', 'name': 'n'}]\n",
      "[{'row Number': '3', 'sno': '4', 'name': 'i'}, {'row Number': '4', 'sno': '5', 'name': 'k'}]\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "# importing the required libraries\n",
    "import csv\n",
    "import gspread\n",
    "import pandas as pd\n",
    "from oauth2client.service_account import ServiceAccountCredentials\n",
    "\n",
    "import google.cloud\n",
    "import firebase_admin\n",
    "from firebase_admin import credentials, firestore\n",
    "\n",
    "\n",
    "# defining the scope\n",
    "scope = ['https://spreadsheets.google.com/feeds',\n",
    "         'https://www.googleapis.com/auth/drive']\n",
    "creds = ServiceAccountCredentials.from_json_keyfile_name('clients.json', scope)\n",
    "\n",
    "# authorize the clientsheet \n",
    "client =gspread.authorize(creds)\n",
    "sheet = client.open('script reading')\n",
    "sheet_instance = sheet.get_worksheet(0)\n",
    "records_data = sheet_instance.get_all_records()\n",
    "\n",
    "# view the data\n",
    "records_data\n",
    "\n",
    "records_df = pd.DataFrame.from_dict(records_data).to_csv('output.csv')\n",
    "\n",
    "\n",
    "cred = credentials.Certificate(\"./firebase_pvtkey.json\")\n",
    "# app = firebase_admin.initialize_app(cred)\n",
    "store = firestore.client()\n",
    "doc_ref = store.collection(u'data')\n",
    "\n",
    "\n",
    "file_path = \"./output.csv\"\n",
    "collection_name = \"data\"\n",
    "\n",
    "\n",
    "def batch_data(iterable, n=1):\n",
    "    l = len(iterable)\n",
    "    for ndx in range(0, l, n):\n",
    "        yield iterable[ndx:min(ndx + n, l)]\n",
    "\n",
    "\n",
    "data = []\n",
    "headers = []\n",
    "\n",
    "#reading csv file and storing each index information\n",
    "with open(file_path) as csv_file:\n",
    "    csv_reader = csv.reader(csv_file, delimiter=',')\n",
    "    line_count = 0\n",
    "    for row in csv_reader:\n",
    "        if line_count == 0:\n",
    "            for header in row:\n",
    "                #if any value in header is empty,this will give an error while uploading it to firestore\n",
    "                if header=='':\n",
    "                    header=\"row Number\"\n",
    "                headers.append(header)\n",
    "            line_count += 1\n",
    "        else:\n",
    "            obj = {}\n",
    "            for idx, item in enumerate(row):\n",
    "                obj[headers[idx]] = item\n",
    "            data.append(obj)\n",
    "            line_count += 1\n",
    "    \n",
    "print(line_count)\n",
    "headers\n",
    "\n",
    "#uploading data to firestore\n",
    "for batched_data in batch_data(data, 3):\n",
    "    batch = store.batch()\n",
    "    print(batched_data)\n",
    "    for data_item in batched_data:\n",
    "        doc_ref = store.collection(collection_name).document()\n",
    "        batch.set(doc_ref, data_item)\n",
    "    batch.commit()\n",
    "\n",
    "print('Completed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}