Created
December 24, 2016 03:41
-
-
Save ryansmccoy/83223b6c35da83a019e0c61f9f63f05d to your computer and use it in GitHub Desktop.
import file or files (csv,xlsx) to mongo database
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import sys, os | |
import pandas as pd | |
import pymongo | |
import json | |
import glob | |
from pymongo import MongoClient | |
fullpath_of_file = r'B:\all.csv.xlsx' | |
#files = glob.glob(r'B:\folderoffiles*') | |
DATABASE = 'database' | |
COLLECTION_NAME = os.path.basename(fullpath_of_file).split('.')[0] # returns basname of filename | |
def mongo_import_file_into_db(DATABASE, COLLECTION_NAME, fullpath_of_file): | |
client = pymongo.MongoClient('localhost', 27017) | |
db_client = client[DATABASE] | |
db = db_client[COLLECTION_NAME] | |
data = pd.read_excel(fullpath_of_file, encoding = 'latin1') | |
#data = pd.read_csv(fullpath_of_file, low_memory = False, encoding = 'utf-8', error_bad_lines = False) | |
data_json = json.loads(data.to_json(orient='records')) | |
db.insert(data_json, check_keys=False) | |
mongo_import_file_into_db(DATABASE, COLLECTION_NAME, fullpath_of_file) | |
#for file in files: | |
# COLLECTION_NAME, fullpath_of_file, DATABASE = os.path.basename(file).split('.')[0], file, DATABASE | |
# mongo_import_file_into_db(DATABASE, COLLECTION_NAME, fullpath_of_file) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment