Skip to content

Instantly share code, notes, and snippets.

@astewart-twist
Last active February 22, 2016 19:10
Show Gist options
  • Save astewart-twist/4af4b4d30c83c0fce438 to your computer and use it in GitHub Desktop.
Save astewart-twist/4af4b4d30c83c0fce438 to your computer and use it in GitHub Desktop.
Write db CSV to S3
# Write dataframe to buffer
csv_buffer = StringIO()
df.to_csv(csv_buffer, index=False)
# Upload CSV to S3
s3_key = 'test.csv'
s3_resource = aws_session.resource('s3')
s3_resource.Object(s3_bucket, s3_key).put(Body=csv_buffer.getvalue())
CREATE OR REPLACE FUNCTION db_to_csv(path TEXT) RETURNS void AS $$
declare
tables RECORD;
statement TEXT;
begin
FOR tables IN
SELECT (table_schema || '.' || table_name) AS schema_table
FROM information_schema.tables t INNER JOIN information_schema.schemata s
ON s.schema_name = t.table_schema
WHERE t.table_schema NOT IN ('pg_catalog', 'information_schema', 'configuration')
ORDER BY schema_table
LOOP
statement := 'COPY ' || tables.schema_table || ' TO ''' || path || '/' || tables.schema_table || '.csv' ||''' DELIMITER '';'' CSV HEADER';
EXECUTE statement;
END LOOP;
return;
end;
$$ LANGUAGE plpgsql;
SELECT db_to_csv('/home/user/dir');
-- this will create one csv file per table, in /home/user/dir/
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment