Created
June 5, 2024 10:45
-
-
Save phuthuy44/3602a8796da6d46667f7a1230745497e to your computer and use it in GitHub Desktop.
When you have a lot of ".csv" file and they are in big folder. You can use this script what can support you to load data from .csv to PostgreSQL
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import psycopg2 | |
import csv | |
import os | |
DB_host ="localhost" | |
DB_name = "data-sts" | |
DB_user ="postgres" | |
DB_pass ="1234" | |
DB_port = "5432" | |
# Establish a connection to the PostgreSQL database | |
conn = psycopg2.connect( | |
host=DB_host, | |
database=DB_name, | |
user=DB_user, | |
password=DB_pass, | |
port=DB_port, | |
) | |
# create a cursor object | |
cur = conn.cursor() | |
# Directory containing CSV files | |
DIRECTORY_PATH = "E:/2023_processed" | |
for filename in os.listdir(DIRECTORY_PATH): | |
if filename.endswith(".csv") and "im" in filename: | |
filepath = os.path.join(DIRECTORY_PATH, filename) | |
try: | |
# read data from csv | |
with open(filepath,mode="r",encoding='utf-8') as file : | |
reader = csv.reader(file) | |
# skip the header row | |
header = next(reader) | |
for row in reader: | |
sql=""" | |
insert into "2023".import( | |
"Buyer", "Supplier", "Total Value(USD)", "Products", "Trade Date", | |
"Unit Price(Currency)", "Currency", "Supplier address", "Exchange rate", | |
"Customs Warehouse Name In Vietnamese Port", "DeclarationNO", | |
"Country of Origin", "HS Code", "Export Port", "Quantity", "Quantity Unit", | |
"Incoterms", "Importer Code", "Total Value(Currency)", "Method Payment", | |
"Buyer address", "Import Tax", "Landing port", "Gross Weight(KG)", | |
"Destination country", "FOB(USD)", "CIF(USD)", "Transportation", | |
"Port of loading", "Type_transaction","city_province","zone" | |
) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,%s,%s,%s) | |
""" | |
# Execute the SQL query | |
cur.execute(sql,row) | |
print(f"Successfully processed file: {filename}") | |
except Exception as e: | |
print(f"Error processing file {filename}: {e}") | |
conn.commit() | |
cur.close() | |
conn.close() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment