Geremie

SELECT
FROM my_table

Geremie

import json

schema = [
 {
   "description": "Name of the house",
   "name": "name",
   "type": "STRING",
   "mode": "REQUIRED"
 },
 {

Geremie

!bq load --autodetect \
--source_format NEWLINE_DELIMITED_JSON \
tmp.test_sample_2 gs://<path_to_folder_containing_the_file>test_sample_2.jsonl address_schema.json

Geremie

!bq load --autodetect \
--source_format NEWLINE_DELIMITED_JSON \
tmp.test_sample_2 gs://<path_to_folder_containing_the_file>test_sample_2.csv code:STRING

Geremie

!bq load --autodetect \
--source_format NEWLINE_DELIMITED_JSON \
tmp.test_sample_2 gs://<path_to_folder_containing_the_file>test_sample_2.jsonl

Geremie

bq load --autodetect \
--skip_leading_rows=1 \
tmp.test_sample_1 gs://<path_to_folder_containing_the_file>/test_sample_1.csv

Geremie

def print_haversine():
    from mypythonlib import myfunctions
    x1, y1, x2, y2 = 1, 2, 3, 4
    haversine_distance = myfunctions.haversine(x1, y1, x2, y2)
    print(f'The haversine distance between ({x1}, {y1}) and ({x2}, {y2}) est {haversine_distance}')


dag = DAG('haversine-distance-dag',
          default_args=DEFAULT_ARGS,
          schedule_interval=None,

Geremie

# Deploy the DAG
gsutil cp dag.py $dags_folder

Geremie

# Add artifact registry reader role for service account
gcloud projects add-iam-policy-binding <project_id> \
  --member=serviceAccount:<your_service_account_id>@<project_id>.iam.gserviceaccount.com \
  --role=roles/artifactregistry.reader
# Create key for service account
gcloud iam service-accounts keys create <path_to_key_file> \
    --iam-account=<your_service_account_id>@<project_id>.iam.gserviceaccount.com
# Get the python repository url and authentication credentials
gcloud artifacts print-settings python --project <your_gcp_project> \
  --repository <your_repository_name> --location <your_repository_location> --json-key <path_to_key_file>

Geremie

# Install the private python package
gcloud composer environments update <your_environment_name> \
  --update-pypi-package mypythonlib --location <your_location_name>