gxercavins

-- create t1
CREATE TABLE
  overwrite.t1 (sales INT64,
    sdate DATE)
PARTITION BY
  sdate;
  
-- create t2
CREATE TABLE
  overwrite.t2 (sales INT64,

gxercavins

import argparse, logging
import pandas as pd
from random import choice

import apache_beam as beam
from apache_beam.options.pipeline_options import SetupOptions

import apache_beam.transforms.combiners as combine
import apache_beam.pvalue as pvalue

gxercavins

package org.apache.beam.examples;

import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
import com.google.cloud.bigquery.BigQuery;

gxercavins

package org.apache.beam.examples;

import java.util.Arrays;
import java.util.List;

import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;

gxercavins

#!/usr/bin/env python

# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#

gxercavins

#!/usr/bin/env python

# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#

gxercavins

import argparse
import logging

import apache_beam as beam

from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.options.pipeline_options import PipelineOptions


def prediction_handler(element):

gxercavins

WITH
  authors AS (
  SELECT
    author,
    DATE_TRUNC(DATE(time_ts), MONTH) AS month
  FROM
    `bigquery-public-data.hacker_news.stories`
  WHERE
    author IS NOT NULL
  GROUP BY 1,2)

gxercavins

import argparse, json, logging
import datetime

import apache_beam as beam
import apache_beam.pvalue as pvalue
from apache_beam.io import ReadFromText
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions

gxercavins

import argparse, json, logging

import apache_beam as beam
import apache_beam.pvalue as pvalue
from apache_beam.io import ReadFromText
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions


class EnrichElementsFn(beam.DoFn):