postgresql_to_datahub_rest_tickit.yml

# Purpose: DataHub example recipe for PostgreSQL datasource
# Author: Gary A. Stafford
# Date: March 2022

# see https://datahubproject.io/docs/metadata-ingestion/source_docs/postgres
source:
  type: postgres
  config:
    # Coordinates
    host_port: ${DB_HOST_PORT}
    database: tickit

    # Credentials
    username: ${DB_USERNAME}
    password: ${DB_PASSWORD}

    # Options
    profiling:
      enabled: true

    # Environment
    env: DEV

# see https://datahubproject.io/docs/metadata-ingestion/transformers/#adding-a-set-of-tags
transformers:
  - type: "simple_add_dataset_tags"
    config:
      tag_urns:
        - "urn:li:tag:AWS"
        - "urn:li:tag:${ACCOUNT_ID}"
        - "urn:li:tag:us-east-1"
  - type: "pattern_add_dataset_terms"
    config:
      term_pattern:
        rules:
          ".*users.*": ["urn:li:glossaryTerm:Classification.Sensitive"]
  - type: "simple_add_dataset_ownership"
    config:
      owner_urns:
        - "urn:li:corpuser:Database Administrators"
      ownership_type: "DATAOWNER"

# see https://datahubproject.io/docs/metadata-ingestion/sink_docs/datahub for complete documentation
sink:
  type: "datahub-rest"
  config:
    server: ${DATAHUB_REST_ENDPOINT}

# see https://datahubproject.io/docs/metadata-ingestion/source_docs/reporting_telemetry/
pipeline_name: "postgres-pipeline-tickit"

reporting:
  - type: "datahub"
    config:
      datahub_api:
        server: ${DATAHUB_REST_ENDPOINT}