mik-laj

# This is a basic workflow to help you get started with Actions

name: CI

on:
  push:
    branches: [ "main" ]
  pull_request:
    branches: [ "main" ]
  schedule:

mik-laj

#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#

mik-laj

#!/usr/bin/env bash

set -euo pipefail
set -x

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "${ROOT_DIR}"

# SOURCE_CONFIG_URL="https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations/connectors/source-file/integration_tests/config.json"
SOURCE_IMAGE_NAME="airbyte/source-file"

mik-laj

import json
from pathlib import Path

import requests
from bs4 import BeautifulSoup
import pygments.lexers.html
from pygments.formatters.terminal256 import Terminal256Formatter, TerminalTrueColorFormatter

URL_PATTERN = "https://mamprawowiedziec.pl/tabelka?id_tabelki=MPW%5CSejmMembersWithContacts&variant=&limit={limit}&strona={page}&par%5Bizba%5D=sejm&par%5Bczy_aktualny%5D=true&filtr=&sort=&tryb=tabelka&parametryWidoku%5Bminiwyszukiwarka%5D=true&parametryWidoku%5Bminiwyszukiwarka-placeholder%5D=Szukaj%20pos%C5%82a%2Fpos%C5%82anki"

mik-laj

import functools
import sys
from pathlib import Path
from typing import List
from pprint import pprint
import requests
import semver
import yaml

mik-laj

import logging

import google.auth
from google.api_core.exceptions import AlreadyExists
from google.cloud import datacatalog
from rich.logging import RichHandler

logging.basicConfig(level=logging.INFO, handlers=[RichHandler()])

log = logging.getLogger(__name__)

mik-laj

import json
import sys
from datetime import datetime
from functools import lru_cache

import humanize
import requests
from typing import NamedTuple, Dict

import semver

mik-laj

import codecs
import re
from os import listdir
from os.path import isfile, join, dirname, abspath, basename
from typing import Any, Optional, Set, Dict
import sys

import black
from jinja2 import Template
import jinja2

mik-laj

Dataflow jobs

Java

gcloud builds submit \
    --config=cloudbuild-java.yaml \
    --substitutions=_APACHE_BEAM_VERSION="2.22.0",_PIPELINE_WAIT_UNTIL_FINISH=false

Python

mik-laj

# Dockerfile with gcloud
FROM ${BASE_AIRFLOW_IMAGE}

RUN RUN curl https://sdk.cloud.google.com | bash \
    && echo "source /home/airflow/google-cloud-sdk/path.bash.inc" >> /home/airflow/.bashrc \
    && echo "source /home/airflow/google-cloud-sdk/completion.bash.inc" >> /home/airflow/.bashrc