Rick Power rgpower

## s3_lambda_event_template.md

      
              1 file
            
          
              0 forks
            
          
              0 comments
            
          
              0 stars
            
          
                rgpower
                / s3_lambda_event_template.md
            
            
              Last active
              October 17, 2020 12:14
            
              
                Using an event template json file, generate a bunch lambda trigger events ,to feed into a lambda, for processing using docker via lambci/lambda
              
          
    you have a list of s3 objects in a bucket that need to be imported manually into BigQuery/BQ for backfill etc, named missing.txt that contains items like
exports/project-activity/firehose/2020/09/28/12/project-activity-batch-s3-1-2020-09-28-12-58-57-56b1c279.gz
exports/project-activity/firehose/2020/09/28/13/project-activity-batch-s3-1-2020-09-28-13-14-00-9e633629.gz
exports/project-activity/firehose/2020/09/28/13/project-activity-batch-s3-1-2020-09-28-13-29-01-f3174b15.gz

take a template file like this, say, named event-json.tpl,
{
  "Records": [

  
## manually_run_scheduled_query.md

      
              1 file
            
          
              0 forks
            
          
              0 comments
            
          
              0 stars
            
          
                rgpower
                / manually_run_scheduled_query.md
            
            
              Created
              October 17, 2020 13:10
            
              
                BigQuery: locate and manually run a scheduled query for a specific run datetime
              
          
    BigQuery: locate and manually run a scheduled query for a specific run datetime

$ bq --project_id my_project_id ls --transfer_config --transfer_location=us | grep query-name

  projects/my_project_id/locations/us/transferConfigs/config_id   query-name   scheduled_query   SUCCEEDED  

$ bq mk --transfer_run --run-time="2020-09-30T23:30:00Z" "projects/my_project_id/locations/us/transferConfigs/config_id"

  
## lambda_s3_to_bq_import.js
"use strict";

const GCP_PROJECT = process.env["GCP_PROJECT"];
const BQ_DATASET = process.env["BQ_DATASET"];
const BQ_TABLE = process.env["BQ_TABLE"];

const S3_BUCKET = process.env["S3_BUCKET"];
const S3_PREFIX = process.env["S3_PREFIX"];
const GCP_CREDENTIALS = process.env["GCP_CREDENTIALS"];

## GitUtils.java
package utils.git;

import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.RepositoryBuilder;
import org.eclipse.jgit.revwalk.RevCommit;

import java.io.File;
import java.io.IOException;

## Browserstack.java
/**
  * Implements the Browserstack API for setting test sessions to pass/fail status as desribed in
  * https://www.browserstack.com/docs/automate/selenium/view-test-results/mark-tests-as-pass-fail
  *
  **/
package utils.browserstack;

import com.google.gson.Gson;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;

## lambda_fn.py
"""
Use this to get output that includes the Lambda context RequestId in your CloudWatch logs
This allows you to differentiate the logs of concurrent invocations.

To use in your lambda:

import logging
logging.getLogger().setLevel(logging.INFO)

def handler(event, context):

## parse_iso8601.py
"""
parse iso8601 '2019-11-26T18:26:19Z' format string in python
"""
def from_iso8601(datetime_iso: str):
    return datetime.datetime.strptime(datetime_iso, "%Y-%m-%dT%H:%M:%S%z")

## invoke_lambda_many_times.sh
# invoke (synchronously) a lambda with a bunch of different uid payloads
cat << EOF > user.lst
abc123
def345
... more...uids...
EOF

cat << EOF > sns.json
{
    "Records": [

## backfill.py
from dotenv import load_dotenv

load_dotenv()

import asyncio
import datetime
import functools
import json
import logging
import os

## Firebase.pm
package AnyEvent::Firebase;

use common::sense;
use AnyEvent;
use AnyEvent::HTTP 'http_request';
use Carp 'croak';
use Coro qw(rouse_cb rouse_wait);
use Coro::Semaphore;
use Crypt::JWT 'encode_jwt';
use JSON::XS;
	"use strict";

	const GCP_PROJECT = process.env["GCP_PROJECT"];
	const BQ_DATASET = process.env["BQ_DATASET"];
	const BQ_TABLE = process.env["BQ_TABLE"];

	const S3_BUCKET = process.env["S3_BUCKET"];
	const S3_PREFIX = process.env["S3_PREFIX"];
	const GCP_CREDENTIALS = process.env["GCP_CREDENTIALS"];
	package utils.git;

	import org.eclipse.jgit.api.Git;
	import org.eclipse.jgit.api.errors.GitAPIException;
	import org.eclipse.jgit.lib.Repository;
	import org.eclipse.jgit.lib.RepositoryBuilder;
	import org.eclipse.jgit.revwalk.RevCommit;

	import java.io.File;
	import java.io.IOException;
	/**
	* Implements the Browserstack API for setting test sessions to pass/fail status as desribed in
	* https://www.browserstack.com/docs/automate/selenium/view-test-results/mark-tests-as-pass-fail
	*
	**/
	package utils.browserstack;

	import com.google.gson.Gson;
	import org.apache.http.HttpEntity;
	import org.apache.http.HttpResponse;
	"""
	Use this to get output that includes the Lambda context RequestId in your CloudWatch logs
	This allows you to differentiate the logs of concurrent invocations.

	To use in your lambda:

	import logging
	logging.getLogger().setLevel(logging.INFO)

	def handler(event, context):
	"""
	parse iso8601 '2019-11-26T18:26:19Z' format string in python
	"""
	def from_iso8601(datetime_iso: str):
	return datetime.datetime.strptime(datetime_iso, "%Y-%m-%dT%H:%M:%S%z")
	# invoke (synchronously) a lambda with a bunch of different uid payloads
	cat << EOF > user.lst
	abc123
	def345
	... more...uids...
	EOF

	cat << EOF > sns.json
	{
	"Records": [
	from dotenv import load_dotenv

	load_dotenv()

	import asyncio
	import datetime
	import functools
	import json
	import logging
	import os
	package AnyEvent::Firebase;

	use common::sense;
	use AnyEvent;
	use AnyEvent::HTTP 'http_request';
	use Carp 'croak';
	use Coro qw(rouse_cb rouse_wait);
	use Coro::Semaphore;
	use Crypt::JWT 'encode_jwt';
	use JSON::XS;