Skip to content

Instantly share code, notes, and snippets.

@mshakhomirov
Created October 20, 2019 17:50
Show Gist options
  • Save mshakhomirov/24a26fa5484e7e3cd3ca7a54e24eec15 to your computer and use it in GitHub Desktop.
Save mshakhomirov/24a26fa5484e7e3cd3ca7a54e24eec15 to your computer and use it in GitHub Desktop.
def _load_table_from_json(bucket_name, file_name, tableSchema, tableName):
blob = CS.get_bucket(bucket_name).blob(file_name)
#! source data file format must be outer array JSON:
body = json.loads(blob.download_as_string())
table_id = BQ.dataset(BQ_DATASET).table(tableName)
job_config.source_format = bigquery.SourceFormat.NEWLINE_DELIMITED_JSON,
job_config.write_disposition = 'WRITE_APPEND'
schema = create_schema_from_yaml(tableSchema)
job_config.schema = schema
load_job = BQ.load_table_from_json(
body,
table_id,
job_config=job_config,
)
load_job.result() # Waits for table load to complete.
print("Job finished.")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment