Skip to content

Instantly share code, notes, and snippets.

View tommydangerous's full-sized avatar
🧙
Fire!

DANGerous tommydangerous

🧙
Fire!
110 followers · 51 following
View GitHub Profile
@tommydangerous
tommydangerous / mage_delta-lake_read_battle-history_partitioned.py
Last active May 11, 2023 18:34
from deltalake import DeltaTable
dt = DeltaTable(
# Change this to your unique URI from a previous step
# if you’re using your own AWS credentials.
's3://mage-demo-public/magic-energy-and-battle-history/1337',
storage_options={
'AWS_ACCESS_KEY_ID': '...',
'AWS_SECRET_ACCESS_KEY': '...',
@tommydangerous
tommydangerous / mage_delta-lake_read_battle-history_versioned.py
Created May 11, 2023 18:11
from deltalake import DeltaTable
dt = DeltaTable(
# Change this to your unique URI from a previous step
# if you’re using your own AWS credentials.
's3://mage-demo-public/battle-history-versioned/1337',
storage_options={
'AWS_ACCESS_KEY_ID': '...',
'AWS_SECRET_ACCESS_KEY': '...',
@tommydangerous
tommydangerous / mage_delta-lake_upload_multiple_battle-history.py
Last active May 11, 2023 18:01
from deltalake.writer import write_deltalake
# ['Aiur', 'Eos', 'Gaia', 'Kamigawa', 'Korhal', 'Ravnica']
planets = list(sorted(set(df['planet'].values)))
# Loop through each planet
for planet in planets:
# Select a subset of the battle history data for a single planet
planet_df = df.query(f"`planet` == '{planet}'")
@tommydangerous
tommydangerous / mage_delta-lake_export_data_into_avatar.py
Last active May 11, 2023 18:18
from deltalake.writer import write_deltalake
@data_exporter
def export_data(combined_data, *args, **kwargs):
write_deltalake(
# Change this URI to your own unique URI
's3://mage-demo-public/magic-energy-and-battle-history/1337',
data=combined_data,
mode='overwrite',
@tommydangerous
tommydangerous / mage_delta-lake_combine_magic_energy_and_battle_history.py
Last active May 9, 2023 23:46
from deltalake import DeltaTable
import pandas as pd
@transformer
def transform(magic_energy, *args, **kwargs):
dt = DeltaTable(
# Change this to your unique URI from a previous step
# if you’re using your own AWS credentials.
's3://mage-demo-public/battle-history/1337',
@tommydangerous
tommydangerous / mage_delta-lake_load_magic_energy.py
Last active May 9, 2023 23:30
from deltalake import DeltaTable
@data_loader
def load_data(*args, **kwargs):
dt = DeltaTable(
's3://mage-demo-public/magic-energy/1337',
storage_options={
'AWS_ACCESS_KEY_ID': '...',
'AWS_SECRET_ACCESS_KEY': '...',
@tommydangerous
tommydangerous / mage_delta-lake_read_battle-history.py
Last active May 9, 2023 23:42
from deltalake import DeltaTable
dt = DeltaTable(
# Change this to your unique URI from a previous step
# if you’re using your own AWS credentials.
's3://mage-demo-public/battle-history/1337',
storage_options={
'AWS_ACCESS_KEY_ID': '...',
'AWS_SECRET_ACCESS_KEY': '...',
@tommydangerous
tommydangerous / mage_delta-lake_upload_battle-history.py
Last active May 11, 2023 17:49
from deltalake.writer import write_deltalake
write_deltalake(
# Change this URI to your own unique URI
's3://mage-demo-public/battle-history/1337',
data=df,
mode='overwrite',
overwrite_schema=True,
storage_options={
@tommydangerous
tommydangerous / mage_delta-lake_download_battle-history.py
Last active May 9, 2023 22:52
import pandas as pd
df = pd.read_csv(
'https://raw.githubusercontent.com/mage-ai/datasets/master/battle_history.csv',
)
@tommydangerous
tommydangerous / create_matrix.py
Created June 17, 2021 05:36
create_matrix.py
import numpy
input_arr = numpy.array([
[10, 20, 30],
[40, 50, 60],
])