salvadorgascon

from google.cloud import bigquery
from google.cloud.exceptions import NotFound
from google.api_core.exceptions import Conflict

from shared.loaders.bigquery_constants import BIGQUERY_PROJECT_NAME

print(f'Contando registros en {BIGQUERY_PROJECT_NAME}.{bigquery_dataset_name}.{bigquery_table_name}')

query_count = f'SELECT COUNT(*) as record_count FROM  {BIGQUERY_PROJECT_NAME}.{bigquery_dataset_name}.{bigquery_table_name}'
query_count_job = bigquery_client.query(query_count) 

salvadorgascon

BIGQUERY_PROJECT_NAME = "project-XXXXXX"

salvadorgascon

BIGQUERY_PROJECT_NAME = "project-XXXXXX"

salvadorgascon

from google.cloud import bigquery
from google.cloud.exceptions import NotFound

def BigQueryDatasetExists(bigquery_client, bigquery_dataset_name):
  exists_dataset = False

  print("Checking dataset " + bigquery_dataset_name, '...', end="")
  
  try:
    bigquery_client.get_dataset(bigquery_dataset_name)    

salvadorgascon

BIGQUERY_PROJECT_NAME = "project-XXXXX"

salvadorgascon

BIGQUERY_PROJECT_NAME = "project-XXXXX"

salvadorgascon

import datetime
import os

from PyPDF2 import PdfReader

def PdfTextTransformer(pdf_binary, tmp_path):
  print("Reading PDF")

  filename = datetime.datetime.now().strftime("%Y%m%d%H%M%S")

salvadorgascon

import datetime
import os

from pandas import read_excel

def ExcelDataframeTransformer(excel_binary, tmp_path):
  print("Reading EXCEL")

  filename = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
  print("Saving EXCEL into", tmp_path+'/'+filename+'.xls')

salvadorgascon

import datetime
import os

from pandas import read_csv

def CsvDataframeTransformer(csv_string, tmp_path):
  print("Reading CSV")

  filename = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
  print("Saving CSV into", tmp_path+'/'+filename+'.csv')

salvadorgascon

import os
from pathlib import Path
from google.cloud import storage

# Use Google Cloude IAM to generate valida keys and set permissions
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.join(Path(__file__).parent,"google-keys.json")

print("Connecting to Google Cloud BigQuery ...", end="")
bigquery_client = bigquery.Client()
print("OK")