Atrix1987

---
  version: "3"
  services: 
    service-a: 
      image: "1234.dkr.ecr.ap-southeast-1.amazonaws.com/service-a:production-v328"
      build: "."
      restart: "always"
      volumes: 
        - "~/.aws:/root/.aws"
      ports: 

Atrix1987

<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/" xmlns:moz="http://www.mozilla.org/2006/browser/search/">
<ShortName>GitHub</ShortName>
<Description>Search GitHub</Description>
<InputEncoding>UTF-8</InputEncoding>
<Image width="16" height="16" type="image/x-icon">https://github.com/favicon.ico</Image>
<Url type="text/html" method="get" template="https://github.com/search?q={searchTerms}&ref=opensearch"/>
<moz:SearchForm>https://github.com/search</moz:SearchForm>
</OpenSearchDescription>

Atrix1987

{
  "Comment": "Load Data from Firebase and send it to AWS Cloud Search",
  "StartAt": "CalculateInterval",
  "States": {
    "Get data from Firebase": {
      "Type": "Parallel",
      "Next": "Format Parallel Processed Data",
      "Branches": [
        {
          "StartAt": "Fetch User Data",

Atrix1987

exports.updateFeed = functions.database.ref('/stories/{userId}/{storyId}').onWrite(event => {
  const userId = event.params.userId;
  const storyId = event.params.storyId;

  let followersRef = admin.database().ref('/followers/'+userId);
  if(!event.data.val()){
    //post was deleted
    followersRef.once("value", function(snap) {
      snap.forEach(function(childSnapshot) {
        let followerId = childSnapshot.key;

Atrix1987

SELECT  YR, month, NewUsers, Recurring, Resurrected, NewUsers+Recurring+Resurrected TOTAL FROM (
  SELECT month, YR, SUM(IF(DIFF IS NULL, uc, 0)) NewUsers, SUM(IF(DIFF = 1, uc, 0)) Recurring, SUM(IF(DIFF > 1, uc, 0)) Resurrected FROM (
    SELECT month, DIFF, YR, EXACT_COUNT_DISTINCT(amplitude_id) uc FROM (
      SELECT amplitude_id, month, YR, month-prevMonth AS DIFF FROM (
        SELECT amplitude_id, month, YR, LAG(month,1) OVER (PARTITION BY amplitude_id ORDER BY month ) AS prevMonth FROM (
          SELECT amplitude_id, YEAR(server_upload_time) YR, MONTH(server_upload_time) AS month FROM (
            SELECT amplitude_id, server_upload_time FROM
            	[gproject:dataset_name.amplitude_start_session])
     		GROUP BY 1, 2, 3
     	)

Atrix1987

SELECT
  integer(mnthid/100) YEAR,
  integer(mnthid%100) MONTH,
  NewUsers,
  Recurring,
  Resurrected,
  NewUsers+Recurring+Resurrected TOTAL FROM (
  SELECT
    mnthid,
    NewUsers,

Atrix1987

private static class PrepareTableRow extends DoFn<String, TableRow> {

  private static final long serialVersionUID = -4178491975010026252L;

  private String eventName;

  public PrepareTableRow(String eventName) {
    this.eventName = eventName;
  }

Atrix1987

public static class ProcessRecords extends PTransform<PCollection<String>, PCollection<TableRow>> {

  private static final long serialVersionUID = -1916823923052893270L;

  private String eventName;

  public ProcessRecords(String eventName) {
    this.eventName = eventName;
  }

Atrix1987

//Configuring the options etc

Pipeline p = Pipeline.create(options);
p.apply(TextIO.Read.named("ReadFiles").from(options.getInputFile()).withCompressionType(CompressionType.GZIP))
  .apply(new ProcessRecords(eventType))
  .apply(BigQueryIO.Write.to(table.getTableReference()).withSchema(table.getSchema())
  .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
  .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND));

p.run();

Atrix1987

YD=`date --date="1 days ago" +%Y%m%d`
echo "Downloading files for $YD from amplitude for Sutori"
curl -u <API_KEY>:<api_secret> "https://amplitude.com/api/2/export?start=${YD}T0&end=${YD}T23" >> $YD.zip
echo 'Unzipping the downloaded files'
unzip "$YD.zip" -d "$YD"
echo 'Formatting folder structure'
mv "${YD}/<APP_ID>"/*.gz "$YD"
rmdir "${YD}"/<APP_ID>
echo 'Uploading to GCS'
gsutil -m cp -r "${YD}" gs://<bucket_name>/sources/amplitude_/events/"${YD}"