Skip to content

Instantly share code, notes, and snippets.

@phuthuy44
Created June 5, 2024 10:45
Show Gist options
  • Save phuthuy44/3602a8796da6d46667f7a1230745497e to your computer and use it in GitHub Desktop.
Save phuthuy44/3602a8796da6d46667f7a1230745497e to your computer and use it in GitHub Desktop.
When you have a lot of ".csv" file and they are in big folder. You can use this script what can support you to load data from .csv to PostgreSQL
import psycopg2
import csv
import os
DB_host ="localhost"
DB_name = "data-sts"
DB_user ="postgres"
DB_pass ="1234"
DB_port = "5432"
# Establish a connection to the PostgreSQL database
conn = psycopg2.connect(
host=DB_host,
database=DB_name,
user=DB_user,
password=DB_pass,
port=DB_port,
)
# create a cursor object
cur = conn.cursor()
# Directory containing CSV files
DIRECTORY_PATH = "E:/2023_processed"
for filename in os.listdir(DIRECTORY_PATH):
if filename.endswith(".csv") and "im" in filename:
filepath = os.path.join(DIRECTORY_PATH, filename)
try:
# read data from csv
with open(filepath,mode="r",encoding='utf-8') as file :
reader = csv.reader(file)
# skip the header row
header = next(reader)
for row in reader:
sql="""
insert into "2023".import(
"Buyer", "Supplier", "Total Value(USD)", "Products", "Trade Date",
"Unit Price(Currency)", "Currency", "Supplier address", "Exchange rate",
"Customs Warehouse Name In Vietnamese Port", "DeclarationNO",
"Country of Origin", "HS Code", "Export Port", "Quantity", "Quantity Unit",
"Incoterms", "Importer Code", "Total Value(Currency)", "Method Payment",
"Buyer address", "Import Tax", "Landing port", "Gross Weight(KG)",
"Destination country", "FOB(USD)", "CIF(USD)", "Transportation",
"Port of loading", "Type_transaction","city_province","zone"
) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s,%s,%s)
"""
# Execute the SQL query
cur.execute(sql,row)
print(f"Successfully processed file: {filename}")
except Exception as e:
print(f"Error processing file {filename}: {e}")
conn.commit()
cur.close()
conn.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment