Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save tommydangerous/336d9ec1e519e9f3632511cb4c7e4a6e to your computer and use it in GitHub Desktop.
Save tommydangerous/336d9ec1e519e9f3632511cb4c7e4a6e to your computer and use it in GitHub Desktop.
from deltalake.writer import write_deltalake
# ['Aiur', 'Eos', 'Gaia', 'Kamigawa', 'Korhal', 'Ravnica']
planets = list(sorted(set(df['planet'].values)))
# Loop through each planet
for planet in planets:
# Select a subset of the battle history data for a single planet
planet_df = df.query(f"`planet` == '{planet}'")
# Write to Delta Lake for each planet and keep appending the data
write_deltalake(
# Change this URI to your own unique URI
's3://mage-demo-public/battle-history-versioned/1337',
data=planet_df,
mode='append',
storage_options={
'AWS_REGION': '...',
'AWS_ACCESS_KEY_ID': '...',
'AWS_SECRET_ACCESS_KEY': '...',
'AWS_S3_ALLOW_UNSAFE_RENAME': 'true',
},
)
print(
f'Created table with {len(planet_df.index)} records for planet {planet}.',
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment