structure.md

├── __init__.py
├── api
│   ├── __init__.py
│   ├── main.py - runs the API and defines routes
│   ├── static - all the static assets for the site
│   ├── templates - index.HTML
│   └── wsgi.py - serves the API via gunicorn
├── conf
│   ├── __init__.py
│   ├── config_manager.py - Allows access to config files
│   └── redis_conn.py - Connection to Redis
├── index - Embeddings indexing operations
│   ├── __init__.py
│   ├── index_embeddings.py - Runs index delete and reindexing as a module
│   ├── index_fields.py - Field schema
│   ├── indexer.py - Indexer class that calls Redis and performs indexing
│   ├── parquet_reader.py - Reads embeddings parquet file into embeddings index
│   └── tests
├── load
│   └── load_test.py - Locust load testing
├── model
│   ├── README.md
│   ├── __init__.py
│   ├── generate_embeddings.ipynb - Generates embeddings on a P3 instance on AWS
│   ├── generate_training_data.py - Cleans training data 
│   ├── onnx_embedding_generator.py - Converts model to ONNX for fast inference
│   └── sentence_embedding_pipeline.py - ONNX helper module
├── notebooks - exploratory code
│   ├── 00_viberary_json_eda.ipynb
│   ├── 01_memray_tracing.ipynb
│   ├── 02_parquet_eda.ipynb
│   ├── 03_duckdb_eda.ipynb
│   ├── 04_word2vec.ipynb
│   ├── 05_duckdb_0.7.1.ipynb
│   ├── 06_doc2vec.ipynb
│   ├── 07_bert_sentence_transformer.ipynb
│   ├── 08_bert_sentence_transformer_gpu.ipynb
│   ├── 09_redis_query_tuning.ipynb
│   ├── cbow.ipynb
│   ├── onnx.ipynb
│   └── redis_hset.ipynb
├── search
│   ├── __init__.py
│   ├── knn_search.py - Search inference module
│   └── tests
└── training_data
    └── 20230711_learned_embeddings.snappy