mneedham

# The NYC Taxis Dataset - https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page

pip install parquet-cli

parq data/yellow_tripdata_2022-01.parquet

parq data/yellow_tripdata_2022-01.parquet --schema

parq data/yellow_tripdata_2022-01.parquet --head 10

mneedham

pip install --upgrade -e git+https://github.com/twintproject/twint.git@origin/master#egg=twint
pip install confluent-kafka[avro]

mneedham

import streamlit as st
import duckdb
from streamlit_searchbox import st_searchbox

atp_duck = duckdb.connect('atp.duck.db', read_only=True)

def search_players(search_term):
    query = '''
    SELECT DISTINCT winner_name AS player
    FROM matches

mneedham

SELECT count(*)
FROM 'data/*.parquet';

SELECT * 
FROM 'data/*.parquet' 
LIMIT 10;

DESCRIBE
SELECT * 
FROM 'data/yellow_tripdata_2011-07.parquet';

mneedham

from flask import Flask
from flask import render_template
import csv
import json

app = Flask(__name__)

@app.route('/')
def my_runs():
    runs = []

mneedham

docker run --rm clickhouse/clickhouse-server:24.3 clickhouse-local --query "SELECT * FROM system.contributors ORDER BY name" > contributors_24.3.txt

docker run --rm clickhouse/clickhouse-server:24.4 clickhouse-local --query "SELECT * FROM system.contributors ORDER BY name" > contributors_24.4.txt

./clickhouse --query "

mneedham

import streamlit as st
import json
from sseclient import SSEClient

print("Listening for updates...")
if "messages" in st.session_state:
    print("Closing old connection")
    st.session_state["messages"].resp.close()

url = "http://127.0.0.1:8000/livetext"

mneedham

import { ClickHouseStore } from "@langchain/community/vectorstores/clickhouse";
import { createRetrievalChain } from "langchain/chains/retrieval";
import { OpenAI, OpenAIEmbeddings } from "@langchain/openai";
import { ChatPromptTemplate } from "@langchain/core/prompts";
import { createStuffDocumentsChain } from "langchain/chains/combine_documents";
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";

import { Document } from '@langchain/core/documents'

const openAIApiKey = "sk-xxx"

mneedham

git clone git@github.com:huggingface/text-embeddings-inference.git
cd text-embeddings-inference

cargo install --path router -F candle -F accelerate

model=BAAI/bge-large-en-v1.5
revision=refs/pr/5
text-embeddings-router --model-id $model --revision $revision --port 8080

mneedham

-- Fails because of weird date
CREATE TABLE players AS 
select *
from 'atp_players.csv';

-- all varchar
CREATE TABLE players1 AS 
select *
from read_csv_auto('atp_players.csv', ALL_VARCHAR=TRUE);