This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
-- Define the payments source table | |
CREATE TABLE payments ( | |
payment_id STRING, | |
user_id STRING, | |
amount DECIMAL(10, 2), | |
timestamp TIMESTAMP(3), | |
WATERMARK FOR timestamp AS timestamp - INTERVAL '5' SECOND | |
) WITH ( | |
'connector' = 'kafka', | |
'topic' = 'payments', |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
-- Step 1: Set up environment to access the Paimon catalog | |
SET 'execution.runtime-mode' = 'batch'; | |
SET 'table.dynamic-table-options.enabled' = 'true'; | |
-- Step 2: Define the source table to read LAION Parquet files | |
CREATE TEMPORARY TABLE source_laion ( | |
id STRING, | |
url STRING, | |
text STRING, | |
height INT, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Sample for reading a file asynchronously using libuv | |
// taken from https://www.snip2code.com/Snippet/247423/Sample-for-reading-a-file-asynchronously | |
#include <stdlib.h> | |
#include <stdio.h> | |
#include <string.h> | |
#include <uv.h> | |
static uv_fs_t openReq; | |
static uv_fs_t readReq; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import boto3 | |
import csv | |
import io | |
from botocore.vendored import requests | |
s3Client = boto3.client('s3') | |
demo_query = """SELECT * | |
FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/nyc-taxi/trips_*.gz', 'TabSeparatedWithNames') |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import ast | |
import boto3 | |
from botocore.exceptions import ClientError | |
def stream_s3_file(bucket: str, key: str, file_size: int, chunk_bytes=5000) -> tuple[dict]: | |
"""Streams a S3 file via a generator. | |
Args: | |
bucket (str): S3 bucket | |
key (str): S3 object path |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const { Kafka } = require('kafkajs'); | |
const axios = require('axios'); | |
const fs = require('fs'); | |
const kafka = new Kafka({ | |
clientId: 'my-app', | |
brokers: ['BROKERS'], // Broker list | |
ssl: { | |
rejectUnauthorized: true, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
from clickhouse_driver import Client | |
# Initialize ClickHouse client | |
clickhouse_host = os.environ['CLICKHOUSE_HOST'] | |
clickhouse_port = os.environ['CLICKHOUSE_PORT'] | |
clickhouse_user = os.environ['CLICKHOUSE_USER'] | |
clickhouse_password = os.environ['CLICKHOUSE_PASSWORD'] | |
clickhouse_database = os.environ['CLICKHOUSE_DATABASE'] | |
clickhouse_table = os.environ['CLICKHOUSE_TABLE'] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#pragma once | |
#include "util/types.hpp" | |
#include "util/std.hpp" | |
#include "util/ndarray.hpp" | |
#include "util/collections.hpp" | |
#include "util/rand.hpp" | |
#include "util/hash.hpp" | |
#include "util/assert.hpp" | |
#include "util/bitset.hpp" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
CREATE TABLE kafka_input ( | |
id INT, | |
event_timestamp STRING, | |
message STRING, | |
proctime AS PROCTIME(), | |
eventtime AS TO_TIMESTAMP(event_timestamp), | |
WATERMARK FOR eventtime AS eventtime - INTERVAL '1' SECOND | |
) WITH ( | |
'connector' = 'kafka', |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
SELECT | |
window_start, | |
CEIL(row_num / 50.0) AS bundle, | |
COUNT(*), | |
AVG(value), | |
MAX(value), | |
MIN(value) | |
FROM ( | |
SELECT | |
*, |
NewerOlder