karol-blaszczyk

How to push tagged Docker releases to Google Artifact Registry with a GitHub Action

Here's how I configured a GitHub Action so that a new version issued by GitHub's release interface will build a Dockerfile, tag it with the version number and upload it to Google Artifact Registry.

Before you attempt the steps below, you need the following:

• A GitHub repository that contains a working Dockerfile
• The Google Cloud SDK tool gcloud installed and authenticated

Create a Workload Identity Federation

karol-blaszczyk

%pyspark
import sys
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.transforms import *

# Create a Glue context
glueContext = GlueContext(SparkContext.getOrCreate())

# Create a DynamicFrame using 

karol-blaszczyk

%pyspark
import sys
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.transforms import *

# Create a Glue context
glueContext = GlueContext(SparkContext.getOrCreate())

# Create a DynamicFrame using 

karol-blaszczyk

import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job
## @params: [JOB_NAME]
args = getResolvedOptions(sys.argv, ['JOB_NAME'])
sc = SparkContext()
glueContext = GlueContext(sc)

karol-blaszczyk

## @params: [JOB_NAME]
args = getResolvedOptions(sys.argv,
                          ['JOB_NAME',
                           'db_name',
                           'db_user',
                           'db_password',
                           'db_host'])

karol-blaszczyk

## Connect PG8000 to database
conn = pg8000.connect(database=args['db_name'],user=args['db_user'],password=args['db_password'],host=args['db_host'],port=5432)
cur = conn.cursor()
# ... 
# Example upsert query 
cur.execute("INSERT INTO {} SELECT * FROM {} ON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name".format('campaigns', 'tmp_staging_campaings'))
# ... 
conn.commit()
cur.close()
conn.close()

karol-blaszczyk

def _save_to_database(frame, table_name):
    frame.toDF().write.format("jdbc").mode('overwrite')\
        .option("url", "jdbc:postgresql://{}:5432/{}".format(args['db_host'],args['db_name'])) \
        .option("user", args['db_user']).option("password", args['db_password'])\
        .option("truncate", "true") \
        .option("dbtable", table_name).save()

karol-blaszczyk

import sys
import re
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.dynamicframe import DynamicFrame
from awsglue.context import GlueContext
from awsglue.job import Job
from pyspark.sql.functions import col
from awsglue.transforms import *
import pg8000

karol-blaszczyk

[{:channel_id=>"RNLRC-T16O0F1OEWt5LWfA", :data=>[{"date"=>"2015-01-02", "total"=>0, "value"=>0}, {"date"=>"2015-01-03", "total"=>0, "value"=>0}, {"date"=>"2015-01-04", "total"=>0, "value"=>0}, {"date"=>"2015-01-05", "total"=>0, "value"=>0}, {"date"=>"2015-01-06", "total"=>0, "value"=>0}, {"date"=>"2015-01-07", "total"=>0, "value"=>0}, {"date"=>"2015-01-08", "total"=>0, "value"=>0}, {"date"=>"2015-01-09", "total"=>0, "value"=>0}, {"date"=>"2015-01-10", "total"=>0, "value"=>0}, {"date"=>"2015-01-11", "total"=>0, "value"=>0}, {"date"=>"2015-01-12", "total"=>0, "value"=>0}, {"date"=>"2015-01-13", "total"=>0, "value"=>0}, {"date"=>"2015-01-14", "total"=>0, "value"=>0}, {"date"=>"2015-01-15", "total"=>0, "value"=>0}, {"date"=>"2015-01-16", "total"=>0, "value"=>0}, {"date"=>"2015-01-17", "total"=>0, "value"=>0}, {"date"=>"2015-01-18", "total"=>0, "value"=>0}, {"date"=>"2015-01-19", "total"=>0, "value"=>0}, {"date"=>"2015-01-20", "total"=>0, "value"=>0}, {"date"=>"2015-01-21", "total"=>0, "value"=>0}, {"date"=>"2015-

karol-blaszczyk

# git repo status in prompt functions
# ripped from <https://github.com/twolfson/sexy-bash-prompt/blob/master/.bash_prompt>
# with small changes
#
function _get_git_branch() {
  # On branches, this will return the branch name
  # On non-branches, (no branch)
  _REF="$(git symbolic-ref HEAD 2> /dev/null | sed -e 's/refs\/heads\///')"
  if [[ $_REF != "" ]]; then
    echo $_REF