Joyan9

import requests
import dlt
from dlt.destinations.adapters import bigquery_adapter
from datetime import date

# Define API endpoint and parameters
url = "https://api.open-meteo.com/v1/forecast"

# define a dlt resource that yields data
@dlt.resource(name="hourly_weather", 

Joyan9

# processing/helpers/cleaning.py
def mask_pii(df: DataFrame, col: str, patterns: List[Tuple[str, str]]) -> DataFrame:
    """
    Mask PII in a column using regex patterns.
    """
    for pattern, mask in patterns:
        df = df.withColumn(col, F.regexp_replace(F.col(col), pattern, mask))
    return df
  
 # processing/helpers/dimensions.py

Joyan9

version: 1.0.0
kind: DataContract
id: 7a9bd53e-c1f4-48d0-a822-56138cbc5e93
status: active
name: customer_profiles
apiVersion: v3.0.2
description: "Basic customer profile information for retail application"

# Data owners and stakeholders
owner: