Skip to content

Instantly share code, notes, and snippets.

View mneedham's full-sized avatar

Mark Needham mneedham

View GitHub Profile
@mneedham
mneedham / contributors_local.md
Created May 3, 2024 13:16
Latest ClickHouse Contributors
docker run --rm clickhouse/clickhouse-server:24.3 clickhouse-local --query "SELECT * FROM system.contributors ORDER BY name" > contributors_24.3.txt
docker run --rm clickhouse/clickhouse-server:24.4 clickhouse-local --query "SELECT * FROM system.contributors ORDER BY name" > contributors_24.4.txt
./clickhouse --query "
import streamlit as st
import json
from sseclient import SSEClient
print("Listening for updates...")
if "messages" in st.session_state:
print("Closing old connection")
st.session_state["messages"].resp.close()
url = "http://127.0.0.1:8000/livetext"
@mneedham
mneedham / ingest.mjs
Last active March 21, 2024 14:32
LangChain Example
import { ClickHouseStore } from "@langchain/community/vectorstores/clickhouse";
import { createRetrievalChain } from "langchain/chains/retrieval";
import { OpenAI, OpenAIEmbeddings } from "@langchain/openai";
import { ChatPromptTemplate } from "@langchain/core/prompts";
import { createStuffDocumentsChain } from "langchain/chains/combine_documents";
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
import { Document } from '@langchain/core/documents'
const openAIApiKey = "sk-xxx"
@mneedham
mneedham / 0_documents.json
Last active November 19, 2023 10:14
FastEmbed
{"url": "https://www.bbc.com/news/uk-politics-67296825", "title": "AI summit: Education will blunt AI risk to jobs, says Rishi Sunak - BBC News", "body": ["People should not be worried about the impact of AI on jobs because education reforms will boost skills, Rishi Sunak has said.", "Speaking after the UK's first AI safety summit, the prime minister said the technology would improve the economy in the long term.", "He added that new tools should be seen as a \"co-pilot\" to help people at work, rather than replacing them.", "The government's job should be to improve training, he told reporters.", "Mr Sunak said he recognised there was \"anxiety\" about the impact new AI tools could have on the workplace, but said it would enhance productivity over time. ", "\"We should look at AI much more as a co-pilot than something which is necessary going to replace someone's job. AI is a tool that can help almost anybody do their jobs better, faster, quicker.", "\"My job, the government's job, is to make sure we have a
@mneedham
mneedham / 0_install.sh
Created October 28, 2023 08:30
Hugging Face's Text Embeddings Inference Library
git clone git@github.com:huggingface/text-embeddings-inference.git
cd text-embeddings-inference
cargo install --path router -F candle -F accelerate
model=BAAI/bge-large-en-v1.5
revision=refs/pr/5
text-embeddings-router --model-id $model --revision $revision --port 8080
@mneedham
mneedham / massage.py
Created September 5, 2023 14:27
Quix Streams Process <n> messages
import quixstreams as qx
from quixstreams import StreamConsumer, EventData, CancellationTokenSource, CommitMode
import time
import json
import sys
import threading
import click
def on_event_data_received_handler(stream: StreamConsumer, data: EventData):
global events_consumed
@mneedham
mneedham / 00_pip.bash
Created July 26, 2023 09:58
Confluent Kafka: DeprecationWarning: AvroProducer has been deprecated. Use AvroSerializer instead.
pip install confluent-kafka avro urllib3 requests fastavro
@mneedham
mneedham / 01_enums.py
Last active July 10, 2023 08:24
enums in duckdb
# Dataset: https://www.kaggle.com/datasets/wilmerarltstrmberg/recipe-dataset-over-2m
import duckdb
db1 = duckdb.connect('db1.duck.db')
db2 = duckdb.connect('db2.duck.db')
db1.sql("""
CREATE OR REPLACE TABLE recipes AS
FROM read_csv_auto('recipes_data.csv', header=True)
@mneedham
mneedham / queries.md
Created April 14, 2023 06:06
Intro to Window Functions
@mneedham
mneedham / queries.md
Last active April 14, 2023 05:54
SQL Aggregate vs Aggregate Window Functions