Skip to content

Instantly share code, notes, and snippets.

@j-wil
Created July 12, 2018 20:39
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save j-wil/29154dac8ed32a2658b3a215c04bb676 to your computer and use it in GitHub Desktop.
Save j-wil/29154dac8ed32a2658b3a215c04bb676 to your computer and use it in GitHub Desktop.
Delete large number of rows from very large sql tables
from sqlalchemy import create_engine
from functools import reduce
import minibar
import time
import ast
import os
eng = create_engine(os.getenv('db_config'))
table = x
take = 10000
run = 1000
total = run * take
template = "{i}/{total} {bar} {elapsed}s {eta}" # Progress bar template
print(f"Deleting {total} records")
for i in minibar.bar(range(run), template=template):
# Connect to DB
connection = eng.connect()
# Row IDs to be deleted
sql_id = f"""SELECT id from {table} ORDER BY id desc limit {take}"""
ids = connection.execute(sql_id).fetchall()
resultset = [tuple(row) for row in ids]
list = reduce(lambda x,y: x+y, resultset)
# Delete rows from table
sql = f"""DELETE FROM {table} where id in {list}"""
connection.execute(sql)
# Close connection
connection.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment