danthelion

set -e

DB_FILE_FULL_PATHG=$1
DB_FILE_PARENT_FOLDER=$(dirname $DB_FILE_FULL_PATHG)
DB_FILE_NAME=$(basename $DB_FILE_FULL_PATHG)

ngrok http file://$DB_FILE_PARENT_FOLDER --log=stdout >/dev/null &

sleep 1

danthelion

    client = SchemaRegistryClient(url="http://127.0.0.1:8081")

    class User(BaseModel):
        ts: str
        name: str
        country: str
        date_of_birth: str
    
    for i in range(100):
        data = User(

danthelion

    client = SchemaRegistryClient(url="http://127.0.0.1:8081")

    class User(BaseModel):
        ts: str
        name: str
        country: str
        age: int
    
    for i in range(100):
        data = User(

danthelion

client = SchemaRegistryClient(url="http://127.0.0.1:8081")


class User(BaseModel):
    ts: datetime.datetime
    name: str
    country: str
    age: int

danthelion

for i in range(100):
    data = {
        "ts": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f"),
        "name": fake.name(),
        "country": fake.country(),
        "date-of-birth": str(fake.date_of_birth(tzinfo=None, minimum_age=0, maximum_age=100)),
    }
    producer.send(topic=TOPIC, key=str(i).encode("utf-8"), value=json.dumps(data).encode("utf-8"))
    print(f"Sent message {i} -> {data}")
    sleep(2)

danthelion

import json
import json
import os

from kafka import KafkaConsumer

BOOTSTRAP_SERVERS = (
    "kafka:9092" if os.getenv("RUNTIME_ENVIRONMENT") == "DOCKER" else "localhost:9092"
)

danthelion

TOPIC = "USERS"

def push_messages():
    producer = KafkaProducer(
        bootstrap_servers=BOOTSTRAP_SERVERS,
    )
    fake = Faker()

    for i in range(100):
        data = {

danthelion

version: '3'
services:

  broker:
    image: confluentinc/cp-kafka:7.2.1
    hostname: broker
    container_name: broker
    ports:
      - "9092:9092"
      - "9101:9101"

danthelion

USE iceberg;

CREATE SCHEMA IF NOT EXISTS iris
    WITH (location = 's3a://iris/');

CREATE TABLE IF NOT EXISTS iceberg.iris.iris_parquet
(
    sepal_length DOUBLE,
    sepal_width  DOUBLE,
    petal_length DOUBLE,

danthelion

  trino-coordinator:
    image: 'trinodb/trino:latest'
    hostname: trino-coordinator
    ports:
      - '8080:8080'
    volumes:
      - ./etc:/etc/trino
    networks:
      - trino