Created
July 12, 2018 20:39
-
-
Save j-wil/29154dac8ed32a2658b3a215c04bb676 to your computer and use it in GitHub Desktop.
Delete large number of rows from very large sql tables
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from sqlalchemy import create_engine | |
from functools import reduce | |
import minibar | |
import time | |
import ast | |
import os | |
eng = create_engine(os.getenv('db_config')) | |
table = x | |
take = 10000 | |
run = 1000 | |
total = run * take | |
template = "{i}/{total} {bar} {elapsed}s {eta}" # Progress bar template | |
print(f"Deleting {total} records") | |
for i in minibar.bar(range(run), template=template): | |
# Connect to DB | |
connection = eng.connect() | |
# Row IDs to be deleted | |
sql_id = f"""SELECT id from {table} ORDER BY id desc limit {take}""" | |
ids = connection.execute(sql_id).fetchall() | |
resultset = [tuple(row) for row in ids] | |
list = reduce(lambda x,y: x+y, resultset) | |
# Delete rows from table | |
sql = f"""DELETE FROM {table} where id in {list}""" | |
connection.execute(sql) | |
# Close connection | |
connection.close() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment