gxercavins

import logging
import apache_beam as beam

PROJECT = "PROJECT_ID"
BUCKET = "BUCKET_NAME"
schema = "index:INTEGER,event:STRING"
FIELD_NAMES = ["index","event"]


class CsvToDictFn(beam.DoFn):

gxercavins

package org.apache.beam.examples;

import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;
import com.google.cloud.bigquery.BigQuery;

gxercavins

#!/usr/bin/env python

# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#

gxercavins

import argparse, json, logging, time

import apache_beam as beam
import apache_beam.transforms.window as window
from apache_beam.io import filesystems
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions


class AddWindowingInfoFn(beam.DoFn):

gxercavins

import argparse, json, logging

import apache_beam as beam
import apache_beam.pvalue as pvalue
from apache_beam.io import ReadFromText
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions


class EnrichElementsFn(beam.DoFn):

gxercavins

import argparse, logging, time
import inflect
 
import apache_beam as beam
import apache_beam.transforms.combiners as combine
from apache_beam.transforms.userstate import BagStateSpec
from apache_beam.coders import VarIntCoder
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
 

gxercavins

-- create t1
CREATE TABLE
  overwrite.t1 (sales INT64,
    sdate DATE)
PARTITION BY
  sdate;
  
-- create t2
CREATE TABLE
  overwrite.t2 (sales INT64,

gxercavins

import argparse, logging
import pandas as pd
from random import choice

import apache_beam as beam
from apache_beam.options.pipeline_options import SetupOptions

import apache_beam.transforms.combiners as combine
import apache_beam.pvalue as pvalue

gxercavins

package org.apache.beam.examples;

import java.util.Arrays;
import java.util.List;

import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.options.PipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;

gxercavins

#!/usr/bin/env python

# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#