Skip to content

Instantly share code, notes, and snippets.

@kacperlukawski
Last active December 11, 2023 04:32
Show Gist options
  • Save kacperlukawski/2d3a3225f15a4cc5772cd1c81866340d to your computer and use it in GitHub Desktop.
Save kacperlukawski/2d3a3225f15a4cc5772cd1c81866340d to your computer and use it in GitHub Desktop.
Qdrant tips&tricks
Display the source blob
Display the rendered blob
Raw
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Display the source blob
Display the rendered blob
Raw
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "638182f1",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:06:19.941711Z",
"start_time": "2023-03-13T10:06:19.795076Z"
}
},
"outputs": [],
"source": [
"import config\n",
"import func\n",
"import numpy as np"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "4ecbd8f4",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:06:20.214480Z",
"start_time": "2023-03-13T10:06:19.943458Z"
}
},
"outputs": [],
"source": [
"from tqdm import tqdm\n",
"from qdrant_client import QdrantClient\n",
"from qdrant_client.http import models as rest"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "85aadef5",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:06:20.290708Z",
"start_time": "2023-03-13T10:06:20.215877Z"
}
},
"outputs": [],
"source": [
"client = QdrantClient(\n",
" url=\"http://localhost\",\n",
" prefer_grpc=True,\n",
")"
]
},
{
"cell_type": "markdown",
"id": "af9f344b",
"metadata": {},
"source": [
"# Basic search operation"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "fc982cbf",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:06:24.218818Z",
"start_time": "2023-03-13T10:06:20.293137Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"4.83 ms ± 224 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
]
}
],
"source": [
"%%timeit\n",
"client.search(\n",
" collection_name=config.COLLECTION_NAME,\n",
" query_vector=np.random.random(2048),\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "723b72b0",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:06:26.425025Z",
"start_time": "2023-03-13T10:06:24.220263Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2.69 ms ± 110 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
]
}
],
"source": [
"%%timeit\n",
"client.search(\n",
" collection_name=config.COLLECTION_NAME,\n",
" query_vector=np.random.random(2048),\n",
" with_payload=False,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "7ac91d34",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:06:29.471205Z",
"start_time": "2023-03-13T10:06:26.427088Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"3.74 ms ± 142 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
]
}
],
"source": [
"%%timeit\n",
"client.search(\n",
" collection_name=config.COLLECTION_NAME,\n",
" query_vector=np.random.random(2048),\n",
" search_params=rest.SearchParams(\n",
" hnsw_ef=4,\n",
" ),\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "3b5f4f60",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:06:43.220251Z",
"start_time": "2023-03-13T10:06:29.472733Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"17 ms ± 998 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
]
}
],
"source": [
"%%timeit\n",
"client.search(\n",
" collection_name=config.COLLECTION_NAME,\n",
" query_vector=np.random.random(2048),\n",
" search_params=rest.SearchParams(\n",
" hnsw_ef=1024,\n",
" ),\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "e902df7d",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:11:07.383510Z",
"start_time": "2023-03-13T10:11:00.188189Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"8.97 ms ± 590 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
]
}
],
"source": [
"%%timeit\n",
"client.search(\n",
" collection_name=config.COLLECTION_NAME,\n",
" query_vector=np.random.random(2048),\n",
" search_params=rest.SearchParams(\n",
" hnsw_ef=1024,\n",
" ),\n",
" with_payload=False,\n",
")"
]
},
{
"cell_type": "markdown",
"id": "0786435c",
"metadata": {},
"source": [
"# Search results"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "8cb77fd7",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:11:07.396321Z",
"start_time": "2023-03-13T10:11:07.385605Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"[ScoredPoint(id='5e349d05-69b5-4126-87a5-376adceeed70', version=39, score=0.358357310295105, payload={'prod_name': 'BEANIE HELMET WINTER CAP', 'section_no': 45, 'perceived_colour_value_name': 'Dark', 'graphical_appearance_no': 1010016, 'index_group_name': 'Baby/Children', 'index_name': 'Children Accessories, Swimwear', 'perceived_colour_master_name': 'Blue', 'garment_group_name': 'Accessories', 'department_no': 4212, 'department_name': 'Kids Boy Big Acc', 'product_type_name': 'Hat/beanie', 'perceived_colour_value_id': 4, 'product_group_name': 'Accessories', 'garment_group_no': 1019, 'section_name': 'Kids Outerwear', 'perceived_colour_master_id': 2, 'colour_group_code': 73, 'product_type_no': 75, 'index_group_no': 4, 'index_code': 'J', 'detail_desc': 'Hat in a printed weave with reflective details, a turned-up peak, earflaps and a hook and loop fastening under the chin. Lining in faux fur and fleece.', 'graphical_appearance_name': 'Solid', 'product_code': 701622, 'colour_group_name': 'Dark Blue'}, vector=None),\n",
" ScoredPoint(id='692f4ee4-b907-4bcc-8360-59a4259647fe', version=6, score=0.32496702671051025, payload={'prod_name': 'Mitsy 3p Ct Brazilian', 'perceived_colour_value_name': 'Dark', 'section_no': 61, 'graphical_appearance_no': 1010016, 'index_group_name': 'Ladieswear', 'index_name': 'Lingeries/Tights', 'perceived_colour_master_name': 'Black', 'garment_group_name': 'Under-, Nightwear', 'department_no': 1334, 'section_name': 'Womens Lingerie', 'product_type_name': 'Underwear bottom', 'perceived_colour_value_id': 4, 'garment_group_no': 1017, 'department_name': 'Casual Lingerie', 'product_group_name': 'Underwear', 'index_group_no': 1, 'colour_group_code': 9, 'product_type_no': 286, 'perceived_colour_master_id': 5, 'index_code': 'B', 'detail_desc': 'Brazilian briefs in jersey made from a cotton blend with lace trims, a low waist, lined gusset and high cut at the back.', 'product_code': 688055, 'graphical_appearance_name': 'Solid', 'colour_group_name': 'Black'}, vector=None),\n",
" ScoredPoint(id='2b794935-70e4-4319-94e9-67e7990f75d7', version=41, score=0.3234955370426178, payload={'prod_name': 'Major 7PCS set', 'perceived_colour_value_name': 'Dark', 'section_no': 41, 'graphical_appearance_no': 1010012, 'index_group_name': 'Baby/Children', 'index_name': 'Baby Sizes 50-98', 'perceived_colour_master_name': 'Grey', 'garment_group_name': 'Jersey Fancy', 'department_no': 6545, 'department_name': 'Baby Boy Jersey Fancy', 'product_type_name': 'Garment Set', 'section_name': 'Baby Boy', 'product_group_name': 'Garment Full body', 'garment_group_no': 1005, 'perceived_colour_value_id': 4, 'index_group_no': 4, 'colour_group_code': 8, 'product_type_no': 270, 'perceived_colour_master_id': 12, 'index_code': 'G', 'detail_desc': 'Set in soft cotton. T-shirt in jersey with a press-stud on one shoulder (except in sizes 1½-4Y). Two long-sleeved tops in lightweight, printed sweatshirt fabric with ribbing around the neckline, cuffs and hem. Long-sleeved top in jersey with a press-stud on one shoulder (except in sizes 1½-4Y). Two pairs of joggers in lightweight sweatshirt fabric with an elasticated waist, side pockets and ribbed hems. One pair of joggers in jersey with an elasticated, drawstring waist.', 'graphical_appearance_name': 'Mixed solid/pattern', 'product_code': 885434, 'colour_group_name': 'Dark Grey'}, vector=None)]"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client.search(\n",
" collection_name=config.COLLECTION_NAME,\n",
" query_vector=np.random.random(2048),\n",
" limit=3,\n",
")"
]
},
{
"cell_type": "markdown",
"id": "a3539a6a",
"metadata": {},
"source": [
"# Search filters"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "f7f5e78b",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:17:21.391838Z",
"start_time": "2023-03-13T10:17:21.349557Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"[ScoredPoint(id='5e349d05-69b5-4126-87a5-376adceeed70', version=39, score=0.3498181104660034, payload={'prod_name': 'BEANIE HELMET WINTER CAP', 'section_no': 45, 'perceived_colour_value_name': 'Dark', 'graphical_appearance_no': 1010016, 'index_group_name': 'Baby/Children', 'index_name': 'Children Accessories, Swimwear', 'perceived_colour_master_name': 'Blue', 'garment_group_name': 'Accessories', 'department_no': 4212, 'section_name': 'Kids Outerwear', 'product_type_name': 'Hat/beanie', 'department_name': 'Kids Boy Big Acc', 'product_group_name': 'Accessories', 'garment_group_no': 1019, 'perceived_colour_value_id': 4, 'index_group_no': 4, 'colour_group_code': 73, 'product_type_no': 75, 'perceived_colour_master_id': 2, 'index_code': 'J', 'detail_desc': 'Hat in a printed weave with reflective details, a turned-up peak, earflaps and a hook and loop fastening under the chin. Lining in faux fur and fleece.', 'graphical_appearance_name': 'Solid', 'product_code': 701622, 'colour_group_name': 'Dark Blue'}, vector=None),\n",
" ScoredPoint(id='475e95a7-3cca-4135-b523-8d63ea2e15c7', version=48, score=0.31715720891952515, payload={'prod_name': 'Tiny coin purse', 'section_no': 43, 'perceived_colour_value_name': 'Bright', 'graphical_appearance_no': 1010009, 'index_group_name': 'Baby/Children', 'index_name': 'Children Accessories, Swimwear', 'perceived_colour_master_name': 'Yellow', 'garment_group_name': 'Accessories', 'department_no': 4313, 'section_name': 'Kids Accessories, Swimwear & D', 'product_type_name': 'Bag', 'department_name': 'Girls Small Acc/Bags', 'product_group_name': 'Accessories', 'garment_group_no': 1019, 'perceived_colour_value_id': 5, 'perceived_colour_master_id': 8, 'colour_group_code': 20, 'product_type_no': 66, 'index_group_no': 4, 'index_code': 'J', 'detail_desc': 'Small, glittery shoulder bag with appliqués and a shoulder strap with a press-stud at one end. Lined. Size approx. 3x7x11 cm.', 'graphical_appearance_name': 'Glittering/Metallic', 'product_code': 860521, 'colour_group_name': 'Other Yellow'}, vector=None),\n",
" ScoredPoint(id='9b5deabb-e8e8-4560-91b5-b759850e5d33', version=22, score=0.3116244077682495, payload={'prod_name': 'Esther set 2pcs', 'perceived_colour_value_name': 'Dusty Light', 'section_no': 44, 'graphical_appearance_no': 1010016, 'index_group_name': 'Baby/Children', 'index_name': 'Baby Sizes 50-98', 'perceived_colour_master_name': 'White', 'perceived_colour_value_id': 1, 'department_no': 2950, 'department_name': 'Baby Toys/Acc', 'product_type_name': 'Accessories set', 'section_name': 'Baby Essentials & Complements', 'product_group_name': 'Accessories', 'garment_group_no': 1019, 'garment_group_name': 'Accessories', 'index_group_no': 4, 'colour_group_code': 11, 'product_type_no': 494, 'perceived_colour_master_id': 9, 'index_code': 'G', 'detail_desc': 'Fleece-lined set with a hat and pair of mittens in a soft cable knit. Hat with a faux fur pompom on top, small imitation suede appliqué on the front, earflaps and ties under the chin. Mittens with ribbed cuffs. Sizes 0-2M and 2-6M without thumbs.', 'graphical_appearance_name': 'Solid', 'product_code': 858306, 'colour_group_name': 'Off White'}, vector=None)]"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client.search(\n",
" collection_name=config.COLLECTION_NAME,\n",
" query_vector=np.random.random(2048),\n",
" query_filter=rest.Filter(\n",
" must=[\n",
" rest.FieldCondition(\n",
" key=\"index_group_name\",\n",
" match=rest.MatchValue(\n",
" value=\"Baby/Children\",\n",
" )\n",
" ),\n",
" ],\n",
" ),\n",
" limit=3,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "554483a6",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:17:53.812474Z",
"start_time": "2023-03-13T10:17:43.963004Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"12.3 ms ± 1.49 ms per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
]
}
],
"source": [
"%%timeit\n",
"client.search(\n",
" collection_name=config.COLLECTION_NAME,\n",
" query_vector=np.random.random(2048),\n",
" query_filter=rest.Filter(\n",
" must=[\n",
" rest.FieldCondition(\n",
" key=\"index_group_name\",\n",
" match=rest.MatchValue(\n",
" value=\"Baby/Children\",\n",
" )\n",
" ),\n",
" ],\n",
" ),\n",
" limit=3,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "17236106",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:29:05.469010Z",
"start_time": "2023-03-13T10:29:04.853273Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"UpdateResult(operation_id=51, status=<UpdateStatus.COMPLETED: 'completed'>)"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client.create_payload_index(\n",
" collection_name=config.COLLECTION_NAME,\n",
" field_name=\"index_group_name\",\n",
" field_schema=\"keyword\",\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "fc2a3473",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:29:17.365934Z",
"start_time": "2023-03-13T10:29:17.352517Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"CollectionInfo(status=<CollectionStatus.GREEN: 'green'>, optimizer_status=<OptimizersStatusOneOf.OK: 'ok'>, vectors_count=50001, indexed_vectors_count=50001, points_count=50001, segments_count=5, config=CollectionConfig(params=CollectionParams(vectors=VectorParams(size=2048, distance=<Distance.COSINE: 'Cosine'>), shard_number=1, replication_factor=1, write_consistency_factor=1, on_disk_payload=True), hnsw_config=HnswConfig(m=16, ef_construct=100, full_scan_threshold=10000, max_indexing_threads=0, on_disk=False, payload_m=None), optimizer_config=OptimizersConfig(deleted_threshold=0.2, vacuum_min_vector_number=1000, default_segment_number=0, max_segment_size=None, memmap_threshold=None, indexing_threshold=10000, flush_interval_sec=5, max_optimization_threads=1), wal_config=WalConfig(wal_capacity_mb=32, wal_segments_ahead=0)), payload_schema={'index_group_name': PayloadIndexInfo(data_type=<PayloadSchemaType.KEYWORD: 'keyword'>, params=None, points=50001)})"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client.get_collection(collection_name=config.COLLECTION_NAME)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "3fd00811",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:29:34.220903Z",
"start_time": "2023-03-13T10:29:31.676810Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"3.08 ms ± 206 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
]
}
],
"source": [
"%%timeit\n",
"client.search(\n",
" collection_name=config.COLLECTION_NAME,\n",
" query_vector=np.random.random(2048),\n",
" query_filter=rest.Filter(\n",
" must=[\n",
" rest.FieldCondition(\n",
" key=\"index_group_name\",\n",
" match=rest.MatchValue(\n",
" value=\"Baby/Children\",\n",
" )\n",
" ),\n",
" ],\n",
" ),\n",
" limit=3,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "d5146af2",
"metadata": {
"ExecuteTime": {
"end_time": "2023-03-13T10:30:38.436517Z",
"start_time": "2023-03-13T10:30:38.418193Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"[ScoredPoint(id='5e349d05-69b5-4126-87a5-376adceeed70', version=39, score=0.39263033866882324, payload={'prod_name': 'BEANIE HELMET WINTER CAP', 'perceived_colour_value_name': 'Dark', 'section_no': 45, 'graphical_appearance_no': 1010016, 'index_group_name': 'Baby/Children', 'index_name': 'Children Accessories, Swimwear', 'perceived_colour_master_name': 'Blue', 'garment_group_name': 'Accessories', 'department_no': 4212, 'section_name': 'Kids Outerwear', 'product_type_name': 'Hat/beanie', 'department_name': 'Kids Boy Big Acc', 'product_group_name': 'Accessories', 'garment_group_no': 1019, 'perceived_colour_value_id': 4, 'index_group_no': 4, 'colour_group_code': 73, 'product_type_no': 75, 'perceived_colour_master_id': 2, 'index_code': 'J', 'detail_desc': 'Hat in a printed weave with reflective details, a turned-up peak, earflaps and a hook and loop fastening under the chin. Lining in faux fur and fleece.', 'graphical_appearance_name': 'Solid', 'product_code': 701622, 'colour_group_name': 'Dark Blue'}, vector=None),\n",
" ScoredPoint(id='9b5deabb-e8e8-4560-91b5-b759850e5d33', version=22, score=0.3261929154396057, payload={'prod_name': 'Esther set 2pcs', 'section_no': 44, 'perceived_colour_value_name': 'Dusty Light', 'graphical_appearance_no': 1010016, 'index_group_name': 'Baby/Children', 'index_name': 'Baby Sizes 50-98', 'perceived_colour_master_name': 'White', 'perceived_colour_value_id': 1, 'department_no': 2950, 'section_name': 'Baby Essentials & Complements', 'product_type_name': 'Accessories set', 'garment_group_name': 'Accessories', 'garment_group_no': 1019, 'product_group_name': 'Accessories', 'department_name': 'Baby Toys/Acc', 'perceived_colour_master_id': 9, 'colour_group_code': 11, 'product_type_no': 494, 'index_group_no': 4, 'index_code': 'G', 'detail_desc': 'Fleece-lined set with a hat and pair of mittens in a soft cable knit. Hat with a faux fur pompom on top, small imitation suede appliqué on the front, earflaps and ties under the chin. Mittens with ribbed cuffs. Sizes 0-2M and 2-6M without thumbs.', 'product_code': 858306, 'graphical_appearance_name': 'Solid', 'colour_group_name': 'Off White'}, vector=None),\n",
" ScoredPoint(id='29883a61-0d5b-480a-aac6-a9e192f1a72e', version=14, score=0.3254295587539673, payload={'prod_name': 'Malte set', 'section_no': 44, 'perceived_colour_value_name': 'Dusty Light', 'graphical_appearance_no': 1010022, 'index_group_name': 'Baby/Children', 'index_name': 'Baby Sizes 50-98', 'perceived_colour_master_name': 'Pink', 'garment_group_name': 'Accessories', 'department_no': 2950, 'section_name': 'Baby Essentials & Complements', 'product_type_name': 'Hat/beanie', 'perceived_colour_value_id': 1, 'garment_group_no': 1019, 'product_group_name': 'Accessories', 'department_name': 'Baby Toys/Acc', 'perceived_colour_master_id': 4, 'colour_group_code': 51, 'product_type_no': 75, 'index_group_no': 4, 'index_code': 'G', 'detail_desc': 'Set with a pair of mittens and a hat with earflaps in a fine knit. Hat with a pompom on top, ties under the chin and a fleece lining. Mittens with foldover ribbing at the top. Size 0-2M and 2-6M without thumbs.', 'graphical_appearance_name': 'Jacquard', 'product_code': 790897, 'colour_group_name': 'Light Pink'}, vector=None)]"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"client.search(\n",
" collection_name=config.COLLECTION_NAME,\n",
" query_vector=np.random.random(2048),\n",
" query_filter=rest.Filter(\n",
" must=[\n",
" rest.FieldCondition(\n",
" key=\"index_group_name\",\n",
" match=rest.MatchValue(\n",
" value=\"Baby/Children\",\n",
" )\n",
" ),\n",
" ],\n",
" ),\n",
" limit=3,\n",
")"
]
},
{
"cell_type": "markdown",
"id": "0155b23b",
"metadata": {},
"source": [
"More about payload indexing: https://qdrant.tech/documentation/indexing/"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ff9fda4f",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Display the source blob
Display the rendered blob
Raw
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Display the source blob
Display the rendered blob
Raw
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment