Skip to content

Instantly share code, notes, and snippets.

@ryansmccoy
Created December 24, 2016 03:41
Show Gist options
  • Save ryansmccoy/83223b6c35da83a019e0c61f9f63f05d to your computer and use it in GitHub Desktop.
Save ryansmccoy/83223b6c35da83a019e0c61f9f63f05d to your computer and use it in GitHub Desktop.
import file or files (csv,xlsx) to mongo database
import sys, os
import pandas as pd
import pymongo
import json
import glob
from pymongo import MongoClient
fullpath_of_file = r'B:\all.csv.xlsx'
#files = glob.glob(r'B:\folderoffiles*')
DATABASE = 'database'
COLLECTION_NAME = os.path.basename(fullpath_of_file).split('.')[0] # returns basname of filename
def mongo_import_file_into_db(DATABASE, COLLECTION_NAME, fullpath_of_file):
client = pymongo.MongoClient('localhost', 27017)
db_client = client[DATABASE]
db = db_client[COLLECTION_NAME]
data = pd.read_excel(fullpath_of_file, encoding = 'latin1')
#data = pd.read_csv(fullpath_of_file, low_memory = False, encoding = 'utf-8', error_bad_lines = False)
data_json = json.loads(data.to_json(orient='records'))
db.insert(data_json, check_keys=False)
mongo_import_file_into_db(DATABASE, COLLECTION_NAME, fullpath_of_file)
#for file in files:
# COLLECTION_NAME, fullpath_of_file, DATABASE = os.path.basename(file).split('.')[0], file, DATABASE
# mongo_import_file_into_db(DATABASE, COLLECTION_NAME, fullpath_of_file)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment