Mike Shakhomirov mshakhomirov
mshakhomirov
/ your-google-project-12345.json
Last active
October 26, 2019 17:00
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "type": "service_account", | |
| "project_id": "your-google-project", | |
| "private_key_id": "§1234567890d", | |
| "private_key": "-----BEGIN PRIVATE KEY-----\nasdfghjklh\n9owGzasdasdasdM61t4s0ww3xKXHigrV\nrTpRDaL5vbxcwedewfeywTKxtE47xq/Z3/asdasdasdasdas++80u9C8igWh\n/zxcvbn\nr12365876098Mm\nsqwertyuioTldH\nQFgmmSLvAgMBAewsrdtfyguYM0lX0\n9+GenzresdtfyguhijkrBYJ41XYJcLDBKpnGg4j1AJp17ztl\asdsad\nKZMXAqYcCFtawQ9pG1v/+1hFd1rMC24WnwnXPdmcXZXD0XlmxwdSctJ/UOJ74One\nDofhBx4/asd+b3bEGFhdCQtkRQQucR5D6875gxh+I7t2TlLwpCsxIiS\nthIGvR+bstrWo21gSQTSBatGQLVVJZjC9ZfehiT+bQKBgQC/HOzeTKMKWLSY0BGO\noPXL/jJCkIYid+LJ7i30xmzyspdL13T3sRdNSASK8gLBAo/wT3vRYoNceqiagzQC\n9/H3B0baYVk67Z4GaLLY4Vkohxmey/dplgfCi+SiexYcJI7RysR1DhVE6b5jOMQe\nq0fUx+0d47GRlVcCsstDRPdzAwKBgQC1yVD//RjQITkgZvfLDLlbAbs25fEmegv5\ncTdJ2xEgpSgtpJcUcHnuOulK5q7JlQjLMUZKT8UVkLNegThmrKp1x3bz4DRroEc5\nw+4wbilf0Cxs0uZE6wNS5n4zuueWUc4A6rVFNpQoOq502zJN4AVO/aaXQNRCCv1z\nd1vM7dNWpQKBgQCqZx0g3U5mlE97DdMJzKU9XYrs5TZkl2bEzDV30lEkWW8jtZna\nnLF9O+CpDz67YBB9ojFxshgUpl2dX61JlFDQBVH8t94699H2P |
mshakhomirov
/ clone.sh
Last active
October 27, 2019 18:09
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| $ git clone https://github.com/mshakhomirov/bigquery-etl-tutorial.git | |
| $ cd bigquery-etl-tutorial | |
| $ npm install |
mshakhomirov
/ package.json
Created
October 12, 2019 18:44
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "name": "gcs-transfer", | |
| "version": "0.0.1", | |
| "private": true, | |
| "scripts": {}, | |
| "dependencies": { | |
| "@google-cloud/storage": "^3.3.0", | |
| "async": "^3.1.0", | |
| "aws-sdk": "^2.512.0", | |
| "moment": "^2.17.1", |
mshakhomirov
/ index.js
Created
October 12, 2019 18:45
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 'use strict'; | |
| const AWS = require('aws-sdk'); | |
| const async = require('async'); | |
| const { Storage } = require('@google-cloud/storage'); | |
| AWS.config.update({region: "eu-west-1"}); | |
| let s3 = new AWS.S3(); | |
| const moment = require('moment'); | |
| let config = require('./config.json'); |
mshakhomirov
/ config.json
Created
October 12, 2019 18:46
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "Tables": [ | |
| { | |
| "name" : "daily_table_1" | |
| }, | |
| { | |
| "name" : "daily_table_2" | |
| }, | |
| { | |
| "name" : "daily_table_3" |
mshakhomirov
/ test.js
Created
October 12, 2019 18:47
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| "use strict"; | |
| let lambda = require('./index'); | |
| let event = require('./event.json'); | |
| lambda.handler(event, | |
| { | |
| done : r => console.log("done, result returned:\n", r), | |
| succeed : r => console.log("success, result returned:\n", r), | |
| fail : e => console.log(e) |
mshakhomirov
/ event.json
Created
October 12, 2019 18:48
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "Records": [ | |
| { | |
| "eventVersion": "2.0", | |
| "eventSource": "aws:s3", | |
| "awsRegion": "eu-west-1", | |
| "eventTime": "2017-04-11T09:29:02.255Z", | |
| "eventName": "ObjectCreated:Put", | |
| "userIdentity": { | |
| "principalId": "AWS:AROAJMDLTHKSTD54VVQXA: ingest-lambda" |
mshakhomirov
/ deploy.sh
Created
October 12, 2019 19:00
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env bash | |
| # chmod +x the_file_name | |
| base=${PWD##*/} | |
| zp=$base".zip" | |
| echo $zp | |
| rm -f $zp | |
| zip -r $zp * -x deploy.sh |
mshakhomirov
/ gist:91a75c9ba949a1acd6f54407997a16e7
Created
October 20, 2019 17:20
newline_delimited_json.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| {"id": "1", "first_name": "John", "last_name": "Doe", "dob": "1968-01-22", "addresses": [{"status": "current", "address": "123 First Avenue", "city": "Seattle", "state": "WA", "zip": "11111", "numberOfYears": "1"}, {"status": "previous", "address": "456 Main Street", "city": "Portland", "state": "OR", "zip": "22222", "numberOfYears": "5"}]} | |
| {"id": "2", "first_name": "John", "last_name": "Doe", "dob": "1968-01-22", "addresses": [{"status": "current", "address": "123 First Avenue", "city": "Seattle", "state": "WA", "zip": "11111", "numberOfYears": "1"}, {"status": "previous", "address": "456 Main Street", "city": "Portland", "state": "OR", "zip": "22222", "numberOfYears": "5"}]} |
mshakhomirov
/ standard.json
Created
October 20, 2019 17:22
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| {"id":"1","first_name":"John","last_name":"Doe","dob":"1968-01-22","addresses":[{"status":"current","address":"123 First Avenue","city":"Seattle","state":"WA","zip":"11111","numberOfYears":"1"},{"status":"previous","address":"456 Main Street","city":"Portland","state":"OR","zip":"22222","numberOfYears":"5"}]}, | |
| {"id":"2","first_name":"John","last_name":"Doe","dob":"1968-01-22","addresses":[{"status":"current","address":"123 First Avenue","city":"Seattle","state":"WA","zip":"11111","numberOfYears":"1"},{"status":"previous","address":"456 Main Street","city":"Portland","state":"OR","zip":"22222","numberOfYears":"5"}]} | |
| ] |
OlderNewer