big_query_test.rb

require 'logger'
require 'openssl'
require 'ruby-prof'
require 'byebug'
require 'signet/oauth_2/client'
require 'google/apis/bigquery_v2'

project_id = <project_id>
dataset_id = <big_query_data_set>
table = <big_query_table>
sql = <the_query>
credentials = JSON.parse(File.open('./sa.json', 'rb').read)

logger = Logger.new(STDOUT)
service = ::Google::Apis::BigqueryV2::BigqueryService.new
service.request_options.timeout_sec = 300
service.request_options.open_timeout_sec = 300
service.authorization = ::Signet::OAuth2::Client.new(
  audience: 'https://accounts.google.com/o/oauth2/token',
  issuer: credentials['client_id'],
  scope: 'https://www.googleapis.com/auth/bigquery',
  signing_key: OpenSSL::PKey::RSA.new(credentials['private_key'], nil),
  token_credential_uri: 'https://accounts.google.com/o/oauth2/token',
)
service.authorization.fetch_access_token!

query_job = {
  use_query_cache: false,
  timeout_ms: 300000,
  query: sql
}

logger.info("Putting first query")
partial = service.query_job(project_id, query_job, {})
job_id = partial.job_reference.job_id
logger.info("Result #{partial.total_rows} for #{job_id}")
while partial.page_token
  # RubyProf.start
  partial = service.get_job_query_results(
    project_id,
    job_id,
    page_token: partial.page_token
  )
  # result = RubyProf.stop
  # byebug
  # printer = RubyProf::FlatPrinter.new(result)
  # printer.print(STDOUT, {})
  logger.info("Retrieved #{partial.rows.size} rows for #{job_id}")
end