Last active
February 1, 2023 21:42
-
-
Save nathancday/40d751db5271659fc4c42fd079b72225 to your computer and use it in GitHub Desktop.
Companion code for the OSC blogpost "Maximizing Search Relevance with Hyperparameter Optimization"
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#### SETUP #### | |
# 1. Download this GIST as a zip (download icon upper-right in web UI) | |
# 2. Unzip locally and navigate into new folder | |
# 3. Setup virtual Python environment with: `python3 -m venv venv` | |
# 4. Activate new virtual environment with: `source venv/bin/activate` | |
# 5. Install Python dependencies with: `pip install -r requirements.txt` | |
# 5. Run this file with: `python hyperparameter_optimization.py` | |
#### HELPERS #### | |
# Some helper code to hit ES and run a basic recall metric. For production use-cases you will | |
# likely want to multithread the evaluation process. | |
import csv | |
import json | |
import requests | |
import statistics | |
import urllib3 | |
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) | |
# The ES endpoint to hit | |
# Check out es-tmdb project for setting up a local tmdb index: https://github.com/o19s/es-tmdb | |
ES = 'http://localhost:9200/tmdb/_search' | |
# Read in queries and ratings data | |
with open('ratings.json') as src: | |
ratings = json.load(src) | |
with open('queries.json') as src: | |
queries = json.load(src) | |
# Evaluates recall for a given queryId and the associated hits from ES | |
# For this example: recall = (Number of relevant docs from ES query) / (Number of relevant documents we know about) | |
def eval_recall(qid, hits): | |
# Extract doc ID's from es response | |
hit_ids = [x['_id'] for x in hits] | |
# Get doc ID's from ratings with relevance grade > 0 | |
relevant_ids = [k for k,v in ratings[qid].items() if int(v) > 0] | |
# Number of hits that were relevant from ES | |
search_count = len([x for x in hit_ids if x in relevant_ids]) | |
# Number of relevant documents for query in ratings | |
ratings_count = min(len(relevant_ids), 10) | |
return (search_count / ratings_count) if ratings_count > 0 else 0 | |
# Runs all rated queries for the given template | |
# | |
# In this example the minimizer will allocate different hyper-parameters | |
# and call this function expecting a metric back. We are using recall here but | |
# you should optimize for whatever makes sense for your search use-case. | |
def run_queries(es_template): | |
global queries | |
metrics = [] | |
# Run each query and evaluate recall against ratings | |
for qid, query in queries.items(): | |
es_template['query']['multi_match']['query'] = query | |
resp = requests.post(ES, json=es_template, verify=False).json() | |
# Metric is negated to work with the minimize process | |
metrics.append(-1 * eval_recall(qid, resp['hits']['hits'])) | |
# Return aggregate mean | |
return statistics.mean(metrics) | |
def log_result(filename, result): | |
iters = [] | |
for ix, x in enumerate(result.x_iters): | |
iters.append([result.func_vals[ix]] + x) | |
with open(f'{filename}.csv', 'w') as out: | |
writer = csv.writer(out) | |
writer.writerow(['eval', 'title_boost', 'overview_boost', 'tagline_boost', 'cast_boost']) | |
writer.writerows(iters) | |
print(f'[{filename}] Optimum recall: ', result.fun) | |
print(f'[{filename}] Optimum hyper parameters: ', result.x) | |
#### HYPERPARAMETER OPTIMIZATION #### | |
from skopt import dummy_minimize, gbrt_minimize, forest_minimize, gp_minimize | |
from skopt.space import Integer | |
from skopt.utils import use_named_args | |
# Dimensions to tune | |
# For this example we're using boost values, but you could tune any query parameters being sent into your search engine | |
space = [ | |
Integer(1, 20, name='title_boost'), | |
Integer(1, 20, name='overview_boost'), | |
Integer(1, 20, name='tagline_boost'), | |
Integer(1, 20, name='cast_boost'), | |
] | |
# The objective function for our minimize operation | |
# | |
# This objective replaces the boosts in an ES query with the allocated hyper parameters from forest_optimize | |
# It will then call run_queries to optimize on the returned metric, which is recall in this example. | |
@use_named_args(space) | |
def objective(**params): | |
es_query = { | |
'_source': ['_id'], | |
'query': { | |
'multi_match': { | |
'query': 'placeholder', | |
'fields': [ | |
f'title^{params["title_boost"]}', | |
f'overview^{params["overview_boost"]}', | |
f'tagline^{params["tagline_boost"]}', | |
f'cast^{params["cast_boost"]}', | |
] | |
} | |
}, | |
'size': 10 | |
} | |
return run_queries(es_query) | |
# The seed values for the dimensions | |
# | |
# You can replace the values here if you're working on optimizing an existing configuration | |
# or if you have a gut feeling about certain parameters. | |
seed = [1, 1, 1, 1] | |
min_args = { | |
'func': objective, | |
'x0': seed, | |
'dimensions': space, | |
'n_calls': 100, | |
'base_estimator': 'ET', | |
'random_state': 540, | |
'verbose': True | |
} | |
dummy_args = min_args.copy() | |
dummy_args.pop('base_estimator') | |
dummy_result = dummy_minimize(**dummy_args) | |
forest_result = forest_minimize(**min_args) | |
gbrt_result = gbrt_minimize(**min_args) | |
gp_result = gp_minimize(**min_args) | |
log_result('dummy', dummy_result) | |
log_result('forest', forest_result) | |
log_result('gbrt', gbrt_result) | |
log_result('gp', gp_result) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"1": "rambo", | |
"2": "rocky", | |
"3": "war games", | |
"4": "crocodile dundee", | |
"5": "matrix", | |
"6": "contact", | |
"7": "space jam", | |
"8": "battlestar galactica", | |
"9": "her", | |
"10": "jobs", | |
"11": "social network", | |
"12": "rocky horror", | |
"13": "shawshank redemption", | |
"14": "french connection", | |
"15": "star wars new hope", | |
"16": "lego batman", | |
"17": "crouching tiger hidden dragon", | |
"18": "fight club", | |
"19": "die hard", | |
"20": "last temptation of christ", | |
"21": "tree of life", | |
"22": "murray saves christmas", | |
"23": "black swan", | |
"24": "emma", | |
"25": "mad max fury road", | |
"26": "kung fury", | |
"27": "mr smith goes to washington", | |
"28": "kill bill", | |
"29": "the godfather", | |
"30": "pulp fiction", | |
"31": "christmas vacation", | |
"32": "psycho", | |
"33": "life is beautiful", | |
"34": "dark knight", | |
"35": "the green mile", | |
"36": "forrest gump", | |
"37": "top gun", | |
"38": "how to train your dragon", | |
"39": "how to train your dragon 2", | |
"40": "star wars" | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"1": { | |
"7555": "4", | |
"1370": "3", | |
"1369": "3", | |
"13258": "2", | |
"1368": "4", | |
"31362": "1", | |
"61410": "1", | |
"319074": "0", | |
"10296": "0", | |
"35868": "0", | |
"131457": "0", | |
"94794": "0", | |
"169869": "0", | |
"34561": "0", | |
"13763": "0", | |
"62414": "0", | |
"21989": "0", | |
"210577": "0", | |
"56949": "0", | |
"79401": "0", | |
"117942": "0", | |
"223195": "0", | |
"70": "0", | |
"22777": "0", | |
"43189": "0", | |
"318": "0", | |
"39829": "0", | |
"208982": "1", | |
"250761": "1", | |
"801": "1", | |
"78999": "1", | |
"600": "1", | |
"26397": "0", | |
"45046": "0", | |
"271110": "0", | |
"59401": "0", | |
"79773": "0", | |
"321": "0", | |
"11553": "0", | |
"32221": "0", | |
"40082": "0" | |
}, | |
"2": { | |
"1366": "4", | |
"1374": "3", | |
"1371": "3", | |
"1246": "3", | |
"1375": "3", | |
"1367": "3", | |
"60375": "3", | |
"110123": "1", | |
"36685": "1", | |
"17711": "1", | |
"2153": "0", | |
"339403": "0", | |
"21501": "0", | |
"81182": "0", | |
"62414": "0", | |
"21989": "0", | |
"11": "0", | |
"12180": "0", | |
"140607": "0", | |
"1893": "0", | |
"330459": "0", | |
"152601": "0", | |
"17819": "0", | |
"198001": "0", | |
"170430": "0", | |
"18206": "0", | |
"47059": "1", | |
"18215": "0", | |
"70808": "1", | |
"351862": "1", | |
"154019": "1", | |
"70": "1", | |
"22777": "0", | |
"43189": "0", | |
"318": "0", | |
"39829": "0", | |
"1578": "1", | |
"25855": "0", | |
"42012": "1", | |
"32276": "0", | |
"105536": "0" | |
}, | |
"3": { | |
"14154": "3", | |
"365371": "0", | |
"116059": "0", | |
"335988": "0", | |
"354287": "0", | |
"390054": "0", | |
"346672": "0", | |
"369885": "0", | |
"293767": "0", | |
"381288": "0", | |
"860": "4", | |
"9614": "0", | |
"125842": "0", | |
"427760": "0", | |
"4247": "0", | |
"30335": "0", | |
"10207": "0", | |
"24964": "0", | |
"91893": "0", | |
"10951": "0", | |
"126148": "0", | |
"10428": "1", | |
"12699": "1", | |
"136698": "0", | |
"1557": "1", | |
"19433": "1", | |
"10431": "0", | |
"222461": "0", | |
"19855": "0", | |
"208988": "0", | |
"12180": "0", | |
"10253": "0", | |
"11": "0", | |
"26508": "0", | |
"2649": "0", | |
"6917": "0", | |
"20158": "0", | |
"76875": "0", | |
"144910": "0" | |
}, | |
"4": { | |
"9671": "4", | |
"9396": "4", | |
"9290": "3", | |
"248390": "1", | |
"221737": "1", | |
"13022": "0", | |
"10539": "0", | |
"17043": "1", | |
"78402": "1", | |
"25652": "1", | |
"36068": "0", | |
"77680": "0", | |
"6972": "1", | |
"182848": "1", | |
"259311": "1", | |
"293028": "0", | |
"64586": "0", | |
"40466": "1", | |
"104221": "0", | |
"183894": "0", | |
"3573": "0", | |
"12254": "0", | |
"17813": "0", | |
"9835": "0", | |
"70490": "0", | |
"239877": "0", | |
"8995": "0", | |
"47964": "0" | |
}, | |
"5": { | |
"603": "4", | |
"604": "3", | |
"605": "3", | |
"55931": "2", | |
"1857": "0", | |
"10999": "0", | |
"4247": "0", | |
"21874": "0", | |
"181886": "0", | |
"21208": "0", | |
"125607": "0", | |
"56441": "0", | |
"124080": "0", | |
"1487": "0", | |
"72867": "0", | |
"11253": "0", | |
"213110": "0", | |
"13805": "0", | |
"104221": "0", | |
"183894": "0", | |
"3573": "0", | |
"12254": "0", | |
"17813": "0", | |
"73262": "1", | |
"17960": "0", | |
"33068": "0", | |
"28377": "0", | |
"13300": "0", | |
"680": "0", | |
"28131": "0", | |
"37988": "0", | |
"18451": "0", | |
"75404": "0" | |
}, | |
"6": { | |
"686": "4", | |
"7011": "1", | |
"20236": "1", | |
"31337": "1", | |
"199": "1", | |
"21136": "1", | |
"109572": "1", | |
"68730": "1", | |
"340676": "1", | |
"364433": "1", | |
"289318": "1", | |
"15990": "1", | |
"775": "1", | |
"22379": "1", | |
"101609": "0", | |
"2153": "0", | |
"339403": "0", | |
"103": "0", | |
"9677": "0", | |
"38187": "0", | |
"157336": "1", | |
"11504": "0", | |
"37910": "0", | |
"2164": "1", | |
"26978": "1", | |
"61591": "1", | |
"20305": "0", | |
"98355": "1", | |
"16203": "1", | |
"8652": "0", | |
"15142": "0", | |
"120271": "0", | |
"13926": "0", | |
"47500": "0", | |
"97397": "0", | |
"696": "0", | |
"16988": "0", | |
"24810": "0", | |
"61415": "0" | |
}, | |
"7": { | |
"2300": "4", | |
"395992": "0", | |
"365942": "0", | |
"188927": "0", | |
"279096": "0", | |
"332502": "0", | |
"158852": "0", | |
"301728": "0", | |
"243938": "0", | |
"157336": "0", | |
"36373": "0", | |
"7555": "0", | |
"1370": "0", | |
"1369": "0", | |
"1368": "0", | |
"288183": "0", | |
"10474": "1", | |
"105583": "0", | |
"267345": "0", | |
"93856": "0", | |
"12435": "0", | |
"14736": "1", | |
"20737": "1", | |
"267": "0", | |
"46094": "1", | |
"10575": "0", | |
"213985": "0", | |
"162442": "0", | |
"76155": "0", | |
"49001": "0", | |
"245950": "0", | |
"272165": "0", | |
"66118": "0", | |
"73351": "0", | |
"136741": "0" | |
}, | |
"8": { | |
"51999": "0", | |
"24128": "0", | |
"61120": "0", | |
"4584": "0", | |
"315010": "0", | |
"325553": "4", | |
"69315": "3", | |
"105077": "3", | |
"11": "1", | |
"12180": "1", | |
"140607": "1", | |
"1893": "1", | |
"330459": "1", | |
"9876": "0", | |
"34614": "0", | |
"74208": "0", | |
"16384": "0", | |
"1918": "0", | |
"34582": "0", | |
"31546": "0", | |
"99258": "0", | |
"44375": "0", | |
"195617": "0", | |
"143841": "0", | |
"247112": "0", | |
"13371": "0", | |
"77887": "0", | |
"82424": "0", | |
"18238": "0", | |
"105538": "0", | |
"99648": "0", | |
"44479": "0", | |
"138535": "0", | |
"8068": "0", | |
"7085": "0", | |
"43212": "0", | |
"54186": "0", | |
"50479": "0" | |
}, | |
"9": { | |
"152601": "4", | |
"17819": "0", | |
"198001": "0", | |
"170430": "0", | |
"18206": "0", | |
"96769": "0", | |
"86457": "0", | |
"130750": "0", | |
"59527": "0", | |
"5143": "0", | |
"1919": "0", | |
"23967": "0", | |
"53156": "0", | |
"79887": "0", | |
"11931": "0", | |
"45241": "0", | |
"14556": "0", | |
"9374": "0", | |
"66113": "0", | |
"265966": "0", | |
"4032": "1", | |
"11317": "1", | |
"73969": "0", | |
"14279": "0", | |
"54198": "0", | |
"47607": "0", | |
"28687": "0", | |
"42757": "0", | |
"11446": "1", | |
"2026": "0", | |
"12223": "2", | |
"9890": "2", | |
"288772": "0", | |
"200236": "1", | |
"60140": "1" | |
}, | |
"10": { | |
"115782": "4", | |
"321697": "3", | |
"89638": "3", | |
"127929": "0", | |
"47099": "0", | |
"16406": "0", | |
"15302": "0", | |
"41588": "0", | |
"281289": "0", | |
"14773": "0", | |
"55069": "0", | |
"21629": "0", | |
"3293": "0", | |
"186606": "3", | |
"213938": "0", | |
"54559": "0", | |
"13119": "0", | |
"27223": "0", | |
"133469": "0", | |
"1774": "0", | |
"243935": "0", | |
"190754": "0", | |
"10536": "0", | |
"9654": "0", | |
"8848": "0", | |
"186759": "0", | |
"242621": "0", | |
"9263": "0", | |
"145762": "0", | |
"25728": "0", | |
"64945": "0" | |
}, | |
"11": { | |
"37799": "4", | |
"267752": "0", | |
"18467": "0", | |
"78294": "0", | |
"10774": "0", | |
"49980": "0", | |
"38408": "0", | |
"39002": "0", | |
"59387": "0", | |
"28303": "0", | |
"32547": "0", | |
"268735": "0", | |
"328069": "0", | |
"16988": "0", | |
"24664": "0", | |
"25218": "0", | |
"23451": "0", | |
"39750": "0", | |
"11779": "0", | |
"33358": "0", | |
"13856": "0", | |
"29015": "0", | |
"42689": "0", | |
"24772": "0", | |
"24432": "0", | |
"35250": "0", | |
"11235": "0", | |
"148503": "0" | |
}, | |
"12": { | |
"36685": "4", | |
"1374": "0", | |
"1246": "0", | |
"1371": "0", | |
"1375": "0", | |
"1367": "0", | |
"81830": "0", | |
"60375": "0", | |
"110123": "0", | |
"33475": "0", | |
"1366": "0", | |
"13696": "0", | |
"17711": "0", | |
"7443": "0", | |
"300669": "0", | |
"14228": "0", | |
"791": "0", | |
"27997": "0", | |
"16139": "0", | |
"49445": "0", | |
"9802": "0", | |
"37060": "0", | |
"26811": "0", | |
"62764": "0", | |
"11449": "0", | |
"18947": "0", | |
"1584": "0", | |
"13655": "0", | |
"10065": "0", | |
"15423": "0", | |
"65891": "0", | |
"27405": "0", | |
"28036": "0", | |
"28067": "0" | |
}, | |
"13": { | |
"278": "4", | |
"214857": "0", | |
"31781": "0", | |
"295958": "0", | |
"81657": "0", | |
"198130": "0", | |
"46016": "0", | |
"241287": "0", | |
"126056": "0", | |
"78049": "0", | |
"13989": "0", | |
"136418": "0", | |
"38124": "0", | |
"38234": "0", | |
"135714": "0", | |
"220820": "0", | |
"75576": "0", | |
"288162": "0", | |
"166901": "0", | |
"48199": "0", | |
"23020": "0", | |
"48214": "0", | |
"18943": "0", | |
"15670": "0", | |
"94214": "0", | |
"43888": "0", | |
"39514": "0", | |
"18254": "0", | |
"13200": "0", | |
"20126": "0", | |
"15265": "0" | |
}, | |
"14": { | |
"1051": "4", | |
"10711": "3", | |
"197950": "0", | |
"288953": "0", | |
"44379": "0", | |
"59558": "0", | |
"27067": "0", | |
"24166": "0", | |
"19086": "0", | |
"288290": "0", | |
"2011": "0", | |
"266040": "0", | |
"33996": "0", | |
"223519": "0", | |
"236661": "0", | |
"30238": "0", | |
"36095": "0", | |
"61462": "0", | |
"15053": "0", | |
"37845": "0", | |
"367647": "0", | |
"25205": "0", | |
"83403": "0", | |
"12724": "0", | |
"4169": "0", | |
"4266": "0", | |
"397": "0", | |
"273169": "0", | |
"18896": "0", | |
"145902": "0" | |
}, | |
"15": { | |
"278604": "2", | |
"278468": "2", | |
"12180": "2", | |
"20787": "2", | |
"76180": "2", | |
"1895": "3", | |
"1894": "3", | |
"11": "4", | |
"330459": "3", | |
"101609": "1", | |
"81899": "1", | |
"29587": "0", | |
"11008": "0", | |
"140607": "3", | |
"1893": "3", | |
"25872": "0", | |
"19837": "0", | |
"18046": "0", | |
"10521": "0", | |
"5717": "0", | |
"86814": "0", | |
"149362": "0", | |
"59232": "0", | |
"91795": "0", | |
"17359": "0", | |
"24499": "0", | |
"171771": "0", | |
"12771": "0", | |
"34479": "0", | |
"12637": "0", | |
"22599": "0", | |
"44087": "0", | |
"129628": "0", | |
"293820": "0", | |
"222461": "0" | |
}, | |
"16": { | |
"324849": "4", | |
"177271": "3", | |
"322456": "3", | |
"137106": "3", | |
"33427": "2", | |
"45162": "1", | |
"364": "1", | |
"251519": "1", | |
"382322": "1", | |
"321528": "1", | |
"415": "1", | |
"29751": "1", | |
"15805": "1", | |
"17074": "1", | |
"125249": "1", | |
"40662": "1", | |
"366924": "1", | |
"21683": "1", | |
"20077": "1", | |
"93560": "1", | |
"163431": "0", | |
"87017": "0", | |
"27093": "0", | |
"77333": "0", | |
"135368": "0", | |
"836": "0", | |
"144121": "0", | |
"40655": "0", | |
"359002": "0", | |
"268": "1", | |
"2661": "1" | |
}, | |
"17": { | |
"146": "4", | |
"263341": "3", | |
"15860": "1", | |
"214857": "0", | |
"16407": "0", | |
"103173": "0", | |
"81870": "0", | |
"10473": "0", | |
"21521": "0", | |
"75465": "0", | |
"287233": "0", | |
"22752": "0", | |
"220504": "0", | |
"39144": "0", | |
"47626": "0", | |
"2805": "0", | |
"75507": "0", | |
"35868": "0", | |
"27245": "0", | |
"39148": "0", | |
"381284": "0", | |
"445": "0", | |
"12476": "0", | |
"360784": "0", | |
"17031": "0", | |
"1059": "0", | |
"80184": "0", | |
"10191": "0", | |
"82702": "0", | |
"56167": "0", | |
"4613": "0" | |
}, | |
"18": { | |
"550": "4", | |
"1366": "1", | |
"26555": "0", | |
"51976": "0", | |
"248774": "0", | |
"43522": "0", | |
"277622": "0", | |
"293660": "0", | |
"17136": "0", | |
"276120": "0", | |
"110491": "0", | |
"13973": "0", | |
"137145": "0", | |
"5951": "0", | |
"39578": "0", | |
"23683": "0", | |
"213983": "0", | |
"55831": "0", | |
"39254": "0", | |
"197033": "0", | |
"11797": "0", | |
"58151": "0", | |
"85670": "0", | |
"13416": "0", | |
"13376": "0", | |
"62934": "0", | |
"17336": "1", | |
"9599": "0", | |
"8851": "0", | |
"14476": "0", | |
"107781": "0", | |
"2108": "0", | |
"305450": "0", | |
"54266": "0", | |
"345922": "0" | |
}, | |
"19": { | |
"562": "4", | |
"1573": "3", | |
"1572": "3", | |
"1571": "3", | |
"47964": "3", | |
"26244": "0", | |
"7517": "0", | |
"269494": "0", | |
"201414": "0", | |
"90374": "0", | |
"20160": "0", | |
"44103": "0", | |
"4836": "0", | |
"73335": "0", | |
"233487": "0", | |
"23449": "0", | |
"13063": "0", | |
"31046": "0", | |
"289010": "0", | |
"259616": "0", | |
"10258": "0", | |
"51311": "0", | |
"13250": "0", | |
"34599": "0", | |
"37270": "0", | |
"83125": "0", | |
"18603": "0", | |
"577": "0", | |
"53861": "0", | |
"29067": "0", | |
"43708": "0", | |
"42735": "0" | |
}, | |
"20": { | |
"11051": "4", | |
"28169": "1", | |
"23830": "0", | |
"615": "2", | |
"19401": "2", | |
"20268": "0", | |
"47980": "2", | |
"97006": "0", | |
"262904": "2", | |
"96973": "1", | |
"24192": "2", | |
"36226": "1", | |
"12545": "2", | |
"23522": "0", | |
"31510": "2", | |
"199512": "2", | |
"68981": "1", | |
"46756": "0", | |
"213386": "2", | |
"149145": "0", | |
"18408": "0", | |
"52364": "0", | |
"24269": "0", | |
"22309": "0", | |
"80169": "0", | |
"11622": "0", | |
"296065": "0", | |
"111396": "0", | |
"39641": "0", | |
"17870": "0", | |
"18128": "0" | |
}, | |
"21": { | |
"8967": "4", | |
"62394": "0", | |
"51409": "0", | |
"45974": "0", | |
"165911": "0", | |
"30177": "0", | |
"91737": "0", | |
"43008": "0", | |
"37284": "0", | |
"43767": "0", | |
"94706": "0", | |
"117730": "0", | |
"36540": "0", | |
"61080": "0", | |
"72358": "0", | |
"25435": "0", | |
"79042": "0", | |
"36072": "0", | |
"159138": "0", | |
"158711": "0", | |
"7555": "0", | |
"1370": "0", | |
"1369": "0", | |
"1368": "0", | |
"288183": "0", | |
"295058": "0", | |
"291351": "0", | |
"111481": "0", | |
"16029": "0", | |
"7860": "0" | |
}, | |
"22": { | |
"26386": "2", | |
"364067": "3", | |
"9647": "2", | |
"175555": "0", | |
"5825": "0", | |
"10433": "0", | |
"28033": "0", | |
"81393": "1", | |
"21501": "1", | |
"172413": "1", | |
"20785": "1", | |
"46240": "1", | |
"86778": "1", | |
"45697": "1", | |
"13187": "1", | |
"14813": "1", | |
"49742": "1", | |
"850": "1", | |
"13189": "1", | |
"13358": "1", | |
"1366": "0", | |
"1374": "0", | |
"1246": "0", | |
"1367": "0", | |
"1371": "0", | |
"44718": "0", | |
"127451": "0", | |
"87169": "0", | |
"153141": "0", | |
"12493": "0", | |
"81182": "1", | |
"9656": "1", | |
"340402": "1", | |
"16412": "1", | |
"52105": "0", | |
"9871": "0", | |
"27429": "0", | |
"73065": "0", | |
"20077": "0", | |
"192393": "0", | |
"15906": "0", | |
"43532": "0", | |
"36670": "0", | |
"13343": "0" | |
}, | |
"23": { | |
"44214": "4", | |
"29882": "4", | |
"8966": "1", | |
"124461": "0", | |
"27327": "0", | |
"39424": "0", | |
"18239": "0", | |
"95242": "0", | |
"22586": "2", | |
"15016": "2", | |
"28302": "1", | |
"54113": "1", | |
"50619": "0", | |
"54491": "0", | |
"61048": "1", | |
"40440": "0", | |
"24021": "0", | |
"199818": "0", | |
"25757": "0", | |
"39914": "0", | |
"73191": "1", | |
"15167": "0", | |
"13759": "1", | |
"49852": "0", | |
"24696": "0", | |
"22582": "0", | |
"287233": "0", | |
"76891": "0", | |
"39853": "0", | |
"30669": "0", | |
"91292": "0", | |
"11816": "0", | |
"116983": "0", | |
"76346": "0" | |
}, | |
"24": { | |
"104221": "4", | |
"183894": "4", | |
"3573": "4", | |
"12254": "4", | |
"17813": "3", | |
"78646": "0", | |
"12432": "0", | |
"63727": "0", | |
"152584": "0", | |
"77468": "0", | |
"26985": "0", | |
"2528": "0", | |
"23512": "0", | |
"13319": "0", | |
"43798": "1", | |
"51828": "0", | |
"250235": "0", | |
"24869": "0", | |
"253161": "0", | |
"11050": "0", | |
"14154": "0", | |
"254464": "0", | |
"269100": "0", | |
"80215": "0", | |
"5000": "0", | |
"77074": "0", | |
"35066": "0", | |
"42266": "1", | |
"22597": "0", | |
"26191": "0", | |
"106402": "0", | |
"198185": "0", | |
"20873": "0", | |
"541": "0", | |
"1283": "0" | |
}, | |
"25": { | |
"76341": "4", | |
"8810": "3", | |
"9659": "3", | |
"9355": "3", | |
"290999": "1", | |
"37776": "0", | |
"9770": "0", | |
"149172": "0", | |
"27460": "0", | |
"62529": "0", | |
"92667": "0", | |
"27543": "0", | |
"24351": "0", | |
"159615": "0", | |
"26258": "0", | |
"251516": "0", | |
"4148": "0", | |
"228150": "0", | |
"47608": "0", | |
"201086": "0", | |
"239573": "0", | |
"13911": "0", | |
"28036": "0", | |
"9346": "0", | |
"32074": "0", | |
"14615": "0", | |
"12611": "0", | |
"28304": "0", | |
"578": "0", | |
"25699": "0", | |
"56599": "0", | |
"11576": "0", | |
"13752": "0" | |
}, | |
"26": { | |
"251516": "4", | |
"18666": "1", | |
"37776": "0", | |
"149172": "0", | |
"27460": "0", | |
"62529": "0", | |
"11891": "0", | |
"50393": "1", | |
"9470": "1", | |
"92667": "0", | |
"27543": "0", | |
"24351": "0", | |
"159615": "0", | |
"58255": "1", | |
"18725": "1", | |
"15854": "1", | |
"60639": "1", | |
"49444": "1", | |
"228150": "0", | |
"9502": "1", | |
"46147": "0", | |
"86577": "0", | |
"41870": "0", | |
"339428": "0", | |
"9614": "0", | |
"14615": "0", | |
"12611": "0", | |
"28304": "0", | |
"11713": "0", | |
"201086": "0", | |
"841": "0", | |
"15": "0", | |
"244": "0", | |
"254": "0", | |
"10730": "0", | |
"13958": "0", | |
"39276": "0", | |
"31947": "0" | |
}, | |
"27": { | |
"3083": "4", | |
"25058": "2", | |
"266814": "1", | |
"15670": "0", | |
"78140": "0", | |
"787": "0", | |
"45019": "0", | |
"35262": "0", | |
"31309": "0", | |
"162284": "0", | |
"260": "0", | |
"303742": "0", | |
"44383": "0", | |
"77012": "0", | |
"33831": "0", | |
"24197": "0", | |
"101449": "0", | |
"55863": "0", | |
"909": "0", | |
"14434": "0", | |
"9928": "0", | |
"10681": "0", | |
"283995": "0", | |
"62994": "0", | |
"212849": "0", | |
"83735": "0", | |
"10107": "0", | |
"287991": "0", | |
"59744": "0", | |
"27297": "0", | |
"67750": "0", | |
"288153": "0", | |
"5424": "0", | |
"20980": "0", | |
"15292": "0", | |
"9067": "0", | |
"12629": "0", | |
"9804": "0" | |
}, | |
"28": { | |
"393": "3", | |
"24": "4", | |
"37517": "0", | |
"330711": "0", | |
"46691": "0", | |
"318922": "0", | |
"8141": "0", | |
"114888": "0", | |
"190906": "0", | |
"115199": "0", | |
"315": "0", | |
"61943": "0", | |
"28049": "0", | |
"17285": "0", | |
"27261": "0", | |
"13589": "0", | |
"53063": "0", | |
"128276": "0", | |
"167305": "0", | |
"54102": "0", | |
"3083": "0", | |
"25058": "0", | |
"909": "0", | |
"24197": "0", | |
"787": "0", | |
"3080": "0", | |
"99875": "0", | |
"10588": "0", | |
"43580": "0", | |
"20239": "0", | |
"25602": "1", | |
"48707": "1", | |
"22484": "1", | |
"12500": "1", | |
"52009": "1", | |
"79118": "0", | |
"15669": "0", | |
"47448": "0", | |
"76960": "0", | |
"293461": "0" | |
}, | |
"29": { | |
"238": "4", | |
"242": "3", | |
"240": "3", | |
"77403": "0", | |
"70829": "2", | |
"18973": "1", | |
"154779": "0", | |
"45485": "2", | |
"49852": "0", | |
"54102": "0", | |
"13533": "0", | |
"8995": "0", | |
"29887": "0", | |
"62836": "0", | |
"36909": "0", | |
"82866": "0", | |
"75579": "0", | |
"400": "1", | |
"21469": "0", | |
"1374": "0", | |
"12112": "0", | |
"9469": "0", | |
"27511": "0", | |
"279690": "0", | |
"31556": "0", | |
"603": "0", | |
"605": "0", | |
"604": "0", | |
"10999": "0", | |
"957": "0", | |
"110398": "0", | |
"21147": "0", | |
"12721": "0", | |
"220933": "0", | |
"26789": "0", | |
"13200": "0", | |
"39514": "0", | |
"256876": "0", | |
"9626": "0", | |
"51422": "0" | |
}, | |
"30": { | |
"680": "4", | |
"28131": "1", | |
"37988": "0", | |
"18451": "0", | |
"75404": "0", | |
"33470": "0", | |
"25208": "0", | |
"43317": "0", | |
"10349": "0", | |
"91063": "0", | |
"55086": "0", | |
"19562": "0", | |
"20781": "0", | |
"47369": "0", | |
"13166": "0", | |
"1092": "0", | |
"54300": "0", | |
"97251": "0", | |
"15990": "0", | |
"10249": "0", | |
"393": "2", | |
"24": "2", | |
"37517": "0", | |
"330711": "0", | |
"46691": "0", | |
"21501": "0", | |
"81182": "0", | |
"340402": "0", | |
"9656": "0", | |
"16412": "0" | |
}, | |
"31": { | |
"5825": "4", | |
"13673": "1", | |
"77499": "1", | |
"16342": "0", | |
"75132": "1", | |
"249072": "0", | |
"13397": "1", | |
"76452": "0", | |
"81393": "1", | |
"158382": "0", | |
"21501": "1", | |
"11419": "1", | |
"172413": "1", | |
"20785": "1", | |
"46240": "1", | |
"86778": "1", | |
"45697": "1", | |
"13187": "1", | |
"14813": "1", | |
"49742": "1", | |
"88293": "0", | |
"4247": "0", | |
"4248": "0", | |
"131916": "0", | |
"4256": "0", | |
"15379": "0", | |
"15771": "0", | |
"10117": "0", | |
"9422": "0", | |
"308165": "0", | |
"25466": "0", | |
"98399": "0", | |
"15250": "0", | |
"72490": "0", | |
"850": "0", | |
"13368": "1", | |
"51052": "0", | |
"17979": "1", | |
"16938": "0", | |
"340402": "0", | |
"12193": "0" | |
}, | |
"32": { | |
"11252": "4", | |
"539": "4", | |
"12662": "3", | |
"1359": "1", | |
"10576": "3", | |
"27723": "0", | |
"35683": "1", | |
"10726": "1", | |
"11629": "0", | |
"560": "0", | |
"32836": "0", | |
"29103": "0", | |
"42307": "0", | |
"28604": "0", | |
"16092": "0", | |
"79363": "0", | |
"49691": "0", | |
"10890": "0", | |
"112336": "0", | |
"9877": "0", | |
"9614": "0", | |
"125842": "0", | |
"427760": "0", | |
"4247": "1", | |
"30335": "0", | |
"41445": "0", | |
"63100": "0", | |
"2359": "0" | |
}, | |
"33": { | |
"637": "4", | |
"288162": "0", | |
"83860": "0", | |
"226247": "0", | |
"58197": "0", | |
"14002": "0", | |
"219781": "0", | |
"59437": "0", | |
"25228": "0", | |
"669": "0", | |
"32630": "0", | |
"302496": "0", | |
"147729": "0", | |
"75720": "0", | |
"38834": "0", | |
"41659": "0", | |
"40651": "0", | |
"20122": "0", | |
"29483": "0", | |
"6414": "0", | |
"201859": "0", | |
"13982": "0", | |
"66178": "0", | |
"60405": "0", | |
"242076": "0", | |
"85593": "0", | |
"21033": "0", | |
"34299": "0", | |
"38396": "0", | |
"13060": "0", | |
"6522": "0", | |
"10020": "0", | |
"648": "0", | |
"453": "0", | |
"10806": "0", | |
"78657": "0", | |
"61864": "0", | |
"78307": "0", | |
"150540": "0", | |
"678": "0", | |
"419430": "0" | |
}, | |
"34": { | |
"72003": "2", | |
"49026": "3", | |
"155": "4", | |
"142061": "3", | |
"123025": "3", | |
"29751": "2", | |
"114875": "0", | |
"17240": "0", | |
"83233": "0", | |
"82485": "0", | |
"13851": "3", | |
"30584": "0", | |
"577": "0", | |
"124080": "0", | |
"58595": "0", | |
"5765": "0", | |
"37834": "0", | |
"98851": "0", | |
"14919": "2", | |
"213270": "0", | |
"319074": "0", | |
"10296": "0", | |
"45148": "0", | |
"101860": "0", | |
"153158": "0", | |
"11797": "0", | |
"58151": "0", | |
"85670": "0", | |
"20674": "0", | |
"87245": "0", | |
"71701": "0", | |
"10836": "0", | |
"300467": "0", | |
"71945": "0", | |
"30931": "0", | |
"46537": "0", | |
"52627": "0", | |
"11056": "0", | |
"257345": "0" | |
}, | |
"35": { | |
"497": "4", | |
"57537": "0", | |
"252147": "0", | |
"117375": "0", | |
"22734": "0", | |
"8275": "0", | |
"65": "0", | |
"24739": "0", | |
"31005": "0", | |
"61988": "0", | |
"9285": "0", | |
"165534": "0", | |
"13794": "0", | |
"86084": "0", | |
"250666": "0", | |
"28323": "0", | |
"251260": "0", | |
"267188": "0", | |
"71864": "0", | |
"30584": "0", | |
"250503": "0", | |
"7443": "0", | |
"9982": "0", | |
"13343": "0", | |
"78206": "0", | |
"10139": "0", | |
"40974": "0", | |
"79475": "0", | |
"55197": "0", | |
"22972": "0", | |
"337556": "0", | |
"313922": "0", | |
"5876": "0", | |
"12471": "0", | |
"248212": "0", | |
"250769": "0" | |
}, | |
"36": { | |
"482": "0", | |
"479": "0", | |
"11492": "0", | |
"494": "0", | |
"493": "0", | |
"13": "4", | |
"26686": "0", | |
"13155": "0", | |
"43688": "0", | |
"33034": "0", | |
"9614": "0", | |
"125842": "0", | |
"427760": "0", | |
"4247": "0", | |
"30335": "0", | |
"73262": "0", | |
"17960": "0", | |
"33068": "0", | |
"28377": "0", | |
"13300": "0", | |
"49874": "0", | |
"290739": "0", | |
"284793": "0", | |
"57811": "0", | |
"31048": "0", | |
"46689": "0", | |
"73952": "0", | |
"97514": "0", | |
"29717": "0", | |
"126141": "0", | |
"183111": "0" | |
}, | |
"37": { | |
"744": "4", | |
"284296": "0", | |
"8764": "0", | |
"1825": "0", | |
"14629": "0", | |
"9595": "1", | |
"9255": "0", | |
"270005": "0", | |
"10911": "0", | |
"1619": "0", | |
"11967": "0", | |
"11302": "0", | |
"36325": "0", | |
"54551": "0", | |
"11340": "0", | |
"91456": "0", | |
"151960": "0", | |
"326": "0", | |
"12657": "0", | |
"286875": "0", | |
"2609": "0", | |
"18300": "0", | |
"53906": "1", | |
"9359": "0", | |
"82684": "0", | |
"82866": "0", | |
"105795": "0", | |
"191600": "0", | |
"11597": "0", | |
"23805": "0", | |
"56648": "0", | |
"862": "0", | |
"13927": "0", | |
"10750": "0", | |
"3080": "0", | |
"9337": "0" | |
}, | |
"38": { | |
"10191": "4", | |
"82702": "3", | |
"70057": "0", | |
"45054": "0", | |
"9461": "0", | |
"9462": "0", | |
"11114": "1", | |
"3482": "0", | |
"22007": "0", | |
"845": "0", | |
"396535": "0", | |
"11305": "0", | |
"372058": "0", | |
"226448": "0", | |
"38319": "0", | |
"47046": "0", | |
"14799": "0", | |
"41275": "0", | |
"37455": "0", | |
"762": "0", | |
"282268": "0", | |
"56171": "0", | |
"57158": "0", | |
"19713": "0", | |
"8386": "0", | |
"259694": "0", | |
"17403": "0", | |
"213417": "0", | |
"65992": "0", | |
"2049": "0", | |
"26685": "0", | |
"88274": "0", | |
"28871": "0", | |
"18120": "0", | |
"281908": "0", | |
"40860": "0" | |
}, | |
"39": { | |
"82702": "4", | |
"10191": "3", | |
"70057": "0", | |
"45054": "0", | |
"9461": "0", | |
"9462": "0", | |
"11114": "1", | |
"211672": "1", | |
"51831": "0", | |
"229407": "0", | |
"54551": "0", | |
"54559": "0", | |
"11": "0", | |
"140607": "0", | |
"12180": "0", | |
"330459": "0", | |
"1895": "0", | |
"155": "0", | |
"72003": "0", | |
"49026": "0", | |
"29751": "0", | |
"123025": "0", | |
"281908": "0", | |
"18120": "0", | |
"1912": "0", | |
"2049": "0", | |
"26685": "0", | |
"33436": "0", | |
"40222": "0", | |
"17031": "0", | |
"291805": "0", | |
"3482": "0" | |
}, | |
"40": { | |
"11": "4", | |
"140607": "3", | |
"12180": "2", | |
"330459": "3", | |
"1895": "2", | |
"1892": "3", | |
"1891": "3", | |
"85783": "0", | |
"52959": "0", | |
"13475": "1", | |
"209276": "0", | |
"81899": "0", | |
"54138": "0", | |
"188927": "0", | |
"200": "0", | |
"13685": "0", | |
"10207": "0", | |
"13996": "0", | |
"174314": "0", | |
"126148": "0", | |
"325553": "1", | |
"69315": "0", | |
"105077": "0", | |
"66588": "0", | |
"130745": "0", | |
"37079": "0", | |
"126757": "0", | |
"39797": "0", | |
"18112": "0", | |
"43052": "0" | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
certifi==2022.9.14 | |
charset-normalizer==2.1.1 | |
idna==3.4 | |
joblib==1.2.0 | |
numpy==1.21.6 | |
pyaml==21.10.1 | |
PyYAML==6.0 | |
requests==2.28.1 | |
scikit-learn==1.0.2 | |
scikit-optimize==0.9.0 | |
scipy==1.7.3 | |
threadpoolctl==3.1.0 | |
urllib3==1.26.12 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment