Skip to content

Instantly share code, notes, and snippets.

@mshakhomirov
Created October 20, 2019 17:59
Show Gist options
  • Save mshakhomirov/2034df16b543d58f26ae8f46158a899c to your computer and use it in GitHub Desktop.
Save mshakhomirov/2034df16b543d58f26ae8f46158a899c to your computer and use it in GitHub Desktop.
def _load_table_from_dataframe(bucket_name, file_name, tableSchema, tableName):
"""
Source data file must be outer JSON
"""
blob = CS.get_bucket(bucket_name).blob(file_name)
body = json.loads(blob.download_as_string())
table_id = BQ.dataset(BQ_DATASET).table(tableName)
schema = create_schema_from_yaml(tableSchema)
job_config.schema = schema
df = pandas.DataFrame(
body,
# In the loaded table, the column order reflects the order of the
# columns in the DataFrame.
columns=["id", "first_name","last_name","dob","addresses"],
)
df['addresses'] = df.addresses.astype(str)
df = df[['id','first_name','last_name','dob','addresses']]
load_job = BQ.load_table_from_dataframe(
df,
table_id,
job_config=job_config,
)
load_job.result()
print("Job finished.")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment