Skip to content

Instantly share code, notes, and snippets.

@veekaybee
Created February 27, 2024 17:03
Show Gist options
  • Save veekaybee/9fa165f5f41fc57d6e2ccd8c7644ca5d to your computer and use it in GitHub Desktop.
Save veekaybee/9fa165f5f41fc57d6e2ccd8c7644ca5d to your computer and use it in GitHub Desktop.
Display the source blob
Display the rendered blob
Raw
{
"cells": [
{
"cell_type": "code",
"execution_count": 16,
"id": "e47139ca-1021-4864-a565-eacaaca47982",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['CM/codexglue_code2text_javascript', 'piqa', 'EleutherAI/mutual', 'winograd_wsc', 'CM/codexglue_code2text_ruby', 'truthful_qa', 'skt/kobest_v1', 'EleutherAI/logiqa', 'wmt14', 'xcopa', 'CM/codexglue_code2text_php', 'gsm8k', 'hitachi-nlp/FLD.v2', 'glue', 'alexandrainst/m_truthfulqa', 'bigbio/pubmed_qa', 'EleutherAI/race', 'openbookqa', 'BigScienceBiasEval/crows_pairs_multilingual', 'baber/logiqa2', 'nq_open', 'EleutherAI/arithmetic', 'EleutherAI/asdiv', 'allenai/ai2_arc', 'wis-k/instruction-following-eval', 'EleutherAI/lambada_openai', 'web_questions', 'allenai/real-toxicity-prompts', 'alexandrainst/m_hellaswag', 'iwslt2017', 'EleutherAI/wikitext_document_level', 'lambada', 'hellaswag', 'CM/codexglue_code2text_python', 'CM/codexglue_code2text_java', 'social_i_qa', 'EleutherAI/drop', 'allegro/klej-polemo2-in', 'allegro/klej-polemo2-out', 'alexandrainst/m_arc', 'EleutherAI/hendrycks_math', 'juletxara/xstory_cloze', 'allenai/qasper', 'winogrande', 'Muennighoff/babi', 'EleutherAI/pile', 'EleutherAI/sycophancy', 'mc_taco', 'math_qa', 'wmt16', 'EleutherAI/hendrycks_ethics', 'sciq', 'EleutherAI/unscramble', 'trivia_qa', 'EleutherAI/headqa', 'anli', 'qa4mre', 'skg/toxigen-data', 'EleutherAI/coqa', 'swag', 'super_glue', 'GBaker/MedQA-USMLE-4-options-hf', 'CM/codexglue_code2text_go', 'corypaik/prost', 'story_cloze', 'medmcqa']\n"
]
}
],
"source": [
"import os\n",
"import yaml\n",
"from dataclasses import dataclass\n",
"\n",
"# Define the directory containing YAML files\n",
"directory = '/Users/vicki/lm-evaluation-harness/lm_eval/tasks'\n",
"\n",
"# Custom function loader to avoid errors with ! function\n",
"@dataclass\n",
"class Funct:\n",
" funct: str\n",
"\n",
"class Loader(yaml.SafeLoader):\n",
" pass\n",
"\n",
"def construct_funct(loader, node):\n",
" return Funct(loader.construct_scalar(node))\n",
"\n",
"Loader.add_constructor('!function', construct_funct)\n",
"\n",
"\n",
"# List to store dataset names\n",
"dataset_names = []\n",
"\n",
"# Loop through each file in the directory\n",
"for root, dirs, files in os.walk(directory):\n",
" for filename in files:\n",
" # Check if the file ends with \".yaml\" or \".yml\"\n",
" if filename.endswith('.yaml') or filename.endswith('.yml'):\n",
" filepath = os.path.join(root, filename)\n",
" \n",
" # Open the file and load its content as YAML\n",
" with open(filepath, 'r') as file:\n",
" try:\n",
" yaml_content = yaml.load(file, Loader=Loader)\n",
" # Check if 'dataset_name' field exists and add its value to the list\n",
" if 'dataset_path' in yaml_content:\n",
" dataset_path = yaml_content['dataset_path']\n",
" dataset_names.append(dataset_path)\n",
" except yaml.YAMLError as e:\n",
" print(f\"Error reading {filepath}: {e}\")\n",
"\n",
"deduplicated_list = list(set(dataset_names))\n",
"print(deduplicated_list)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "925cf88b-a63b-46a3-8963-c0a622ef5649",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"DownloadConfig(cache_dir=None, force_download=False, resume_download=False, local_files_only=False, proxies=None, user_agent=None, extract_compressed_file=False, force_extract=False, delete_extracted=False, use_etag=True, num_proc=None, max_retries=1, token=None, ignore_url_params=False, storage_options={'hf': {'token': None, 'endpoint': 'https://huggingface.co'}}, download_desc=None)\n",
"wmt14\n",
"None\n",
"None\n",
"False\n"
]
}
],
"source": [
"from datasets.load import HubDatasetModuleFactoryWithScript\n",
"\n",
"# trust = tc('EleutherAI/arithmetic')\n",
"# print(trust)\n",
"\n",
"factory = HubDatasetModuleFactoryWithScript('wmt14', trust_remote_code=False)\n",
"print(factory.download_config)\n",
"print(factory.name)\n",
"print(factory.revision)\n",
"print(factory.dynamic_modules_path)\n",
"print(factory.trust_remote_code)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "c103ecf0-e929-4182-a97f-f399054b6cc1",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Checking medmcqa...\n",
"Checking CM/codexglue_code2text_javascript...\n",
"DatasetModule(module_path='datasets_modules.datasets.piqa.6c611c1a9bf220943c4174e117d3b660859665baf1d43156230116185312d011.piqa', hash='6c611c1a9bf220943c4174e117d3b660859665baf1d43156230116185312d011', builder_kwargs={'base_path': 'https://huggingface.co/datasets/piqa/resolve/main', 'repo_id': 'piqa'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking piqa...\n",
"Loading EleutherAI/mutual requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/mutual...\n",
"DatasetModule(module_path='datasets_modules.datasets.winograd_wsc.0651311f3b6dda14889d9a063030a02458395ee50ab9f41cca4cd5a89c0c3dce.winograd_wsc', hash='0651311f3b6dda14889d9a063030a02458395ee50ab9f41cca4cd5a89c0c3dce', builder_kwargs={'base_path': 'https://huggingface.co/datasets/winograd_wsc/resolve/main', 'repo_id': 'winograd_wsc'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking winograd_wsc...\n",
"Checking CM/codexglue_code2text_ruby...\n",
"Checking truthful_qa...\n",
"Loading skt/kobest_v1 requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking skt/kobest_v1...\n",
"Loading EleutherAI/logiqa requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/logiqa...\n",
"DatasetModule(module_path='datasets_modules.datasets.wmt14.d5cfc45c32d826941d8678bf74c810c2aaa057cdc5544f1e23a5dab8c0407a9f.wmt14', hash='d5cfc45c32d826941d8678bf74c810c2aaa057cdc5544f1e23a5dab8c0407a9f', builder_kwargs={'base_path': 'https://huggingface.co/datasets/wmt14/resolve/main', 'repo_id': 'wmt14'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking wmt14...\n",
"Checking xcopa...\n",
"Checking CM/codexglue_code2text_php...\n",
"Checking gsm8k...\n",
"Checking hitachi-nlp/FLD.v2...\n",
"Checking glue...\n",
"Checking alexandrainst/m_truthfulqa...\n",
"Loading bigbio/pubmed_qa requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking bigbio/pubmed_qa...\n",
"Loading EleutherAI/race requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/race...\n",
"Checking openbookqa...\n",
"Loading BigScienceBiasEval/crows_pairs_multilingual requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking BigScienceBiasEval/crows_pairs_multilingual...\n",
"Loading baber/logiqa2 requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking baber/logiqa2...\n",
"Checking nq_open...\n",
"Loading EleutherAI/arithmetic requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/arithmetic...\n",
"Loading EleutherAI/asdiv requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/asdiv...\n",
"Checking allenai/ai2_arc...\n",
"Checking wis-k/instruction-following-eval...\n",
"Loading EleutherAI/lambada_openai requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/lambada_openai...\n",
"Checking web_questions...\n",
"Checking allenai/real-toxicity-prompts...\n",
"Checking alexandrainst/m_hellaswag...\n",
"DatasetModule(module_path='datasets_modules.datasets.iwslt2017.03ce9110373117c6f6687719f49f269486a8cd49dcad2527993a316cd4b6ad49.iwslt2017', hash='03ce9110373117c6f6687719f49f269486a8cd49dcad2527993a316cd4b6ad49', builder_kwargs={'base_path': 'https://huggingface.co/datasets/iwslt2017/resolve/main', 'repo_id': 'iwslt2017'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking iwslt2017...\n",
"Loading EleutherAI/wikitext_document_level requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/wikitext_document_level...\n",
"Checking lambada...\n",
"DatasetModule(module_path='datasets_modules.datasets.hellaswag.512a66dd8b1b1643ab4a48aa4f150d04c91680da6a4096498a5e5f799623d5ae.hellaswag', hash='512a66dd8b1b1643ab4a48aa4f150d04c91680da6a4096498a5e5f799623d5ae', builder_kwargs={'base_path': 'https://huggingface.co/datasets/hellaswag/resolve/main', 'repo_id': 'hellaswag'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking hellaswag...\n",
"Checking CM/codexglue_code2text_python...\n",
"Checking CM/codexglue_code2text_java...\n",
"DatasetModule(module_path='datasets_modules.datasets.social_i_qa.674d85e42ac7430d3dcd4de7007feaffcb1527c535121e09bab2803fbcc925f8.social_i_qa', hash='674d85e42ac7430d3dcd4de7007feaffcb1527c535121e09bab2803fbcc925f8', builder_kwargs={'base_path': 'https://huggingface.co/datasets/social_i_qa/resolve/main', 'repo_id': 'social_i_qa'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking social_i_qa...\n",
"Loading EleutherAI/drop requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/drop...\n",
"Checking allegro/klej-polemo2-in...\n",
"Checking allegro/klej-polemo2-out...\n",
"Checking alexandrainst/m_arc...\n",
"Loading EleutherAI/hendrycks_math requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/hendrycks_math...\n",
"Loading juletxara/xstory_cloze requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking juletxara/xstory_cloze...\n",
"Loading allenai/qasper requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking allenai/qasper...\n",
"DatasetModule(module_path='datasets_modules.datasets.winogrande.a826c3d3506aefe0e9e9390dcb53271070536586bab95849876b2c1743df56e2.winogrande', hash='a826c3d3506aefe0e9e9390dcb53271070536586bab95849876b2c1743df56e2', builder_kwargs={'base_path': 'https://huggingface.co/datasets/winogrande/resolve/main', 'repo_id': 'winogrande'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking winogrande...\n",
"Checking Muennighoff/babi...\n",
"Loading EleutherAI/pile requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/pile...\n",
"Loading EleutherAI/sycophancy requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/sycophancy...\n",
"DatasetModule(module_path='datasets_modules.datasets.mc_taco.fca37fbe424ae58845baa626b2794046ca67f3b8e85749cdf169a983584f7614.mc_taco', hash='fca37fbe424ae58845baa626b2794046ca67f3b8e85749cdf169a983584f7614', builder_kwargs={'base_path': 'https://huggingface.co/datasets/mc_taco/resolve/main', 'repo_id': 'mc_taco'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking mc_taco...\n",
"DatasetModule(module_path='datasets_modules.datasets.math_qa.67fc1cc5d22b185002c6fd16e19e4d5215eae01fb04d656bed83204ba6ee55ff.math_qa', hash='67fc1cc5d22b185002c6fd16e19e4d5215eae01fb04d656bed83204ba6ee55ff', builder_kwargs={'base_path': 'https://huggingface.co/datasets/math_qa/resolve/main', 'repo_id': 'math_qa'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking math_qa...\n",
"DatasetModule(module_path='datasets_modules.datasets.wmt16.f5dc442f4d1c2cc487cd2d5591af56c03a5f03bb98a3bb92151d015c8c9cb7ad.wmt16', hash='f5dc442f4d1c2cc487cd2d5591af56c03a5f03bb98a3bb92151d015c8c9cb7ad', builder_kwargs={'base_path': 'https://huggingface.co/datasets/wmt16/resolve/main', 'repo_id': 'wmt16'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking wmt16...\n",
"Loading EleutherAI/hendrycks_ethics requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/hendrycks_ethics...\n",
"Checking sciq...\n",
"Loading EleutherAI/unscramble requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/unscramble...\n",
"Checking trivia_qa...\n",
"Loading EleutherAI/headqa requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/headqa...\n",
"Checking anli...\n",
"DatasetModule(module_path='datasets_modules.datasets.qa4mre.3613d898cbc2cc43d8c56d926c72187c367806a021bb39e5c0c82de70ea24c74.qa4mre', hash='3613d898cbc2cc43d8c56d926c72187c367806a021bb39e5c0c82de70ea24c74', builder_kwargs={'base_path': 'https://huggingface.co/datasets/qa4mre/resolve/main', 'repo_id': 'qa4mre'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking qa4mre...\n",
"Loading skg/toxigen-data requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking skg/toxigen-data...\n",
"Loading EleutherAI/coqa requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking EleutherAI/coqa...\n",
"DatasetModule(module_path='datasets_modules.datasets.swag.9640de08cdba6a1469ed3834fcab4b8ad8e38caf5d1ba5e7436d8b1fd067ad4c.swag', hash='9640de08cdba6a1469ed3834fcab4b8ad8e38caf5d1ba5e7436d8b1fd067ad4c', builder_kwargs={'base_path': 'https://huggingface.co/datasets/swag/resolve/main', 'repo_id': 'swag'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking swag...\n",
"DatasetModule(module_path='datasets_modules.datasets.super_glue.bb9675f958ebfee0d5d6dc5476fafe38c79123727a7258d515c450873dbdbbed.super_glue', hash='bb9675f958ebfee0d5d6dc5476fafe38c79123727a7258d515c450873dbdbbed', builder_kwargs={'base_path': 'https://huggingface.co/datasets/super_glue/resolve/main', 'repo_id': 'super_glue'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking super_glue...\n",
"Checking GBaker/MedQA-USMLE-4-options-hf...\n",
"Checking CM/codexglue_code2text_go...\n",
"Loading corypaik/prost requires you to execute the dataset script in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.\n",
"Checking corypaik/prost...\n",
"DatasetModule(module_path='datasets_modules.datasets.story_cloze.45cead0538c3deb72d731a7990e60835c2c9c5d5d5b1e95a7dd47ccf593671e4.story_cloze', hash='45cead0538c3deb72d731a7990e60835c2c9c5d5d5b1e95a7dd47ccf593671e4', builder_kwargs={'base_path': 'https://huggingface.co/datasets/story_cloze/resolve/main', 'repo_id': 'story_cloze'}, builder_configs_parameters=BuilderConfigsParameters(metadata_configs=None, builder_configs=None, default_config_name=None), dataset_infos=None)\n",
"Checking story_cloze...\n"
]
}
],
"source": [
"from datasets.load import HubDatasetModuleFactoryWithScript\n",
"\n",
"trust_remote_code_tasks = []\n",
"\n",
"for task in deduplicated_list:\n",
" try:\n",
" print(f\"Checking {factory.name}...\")\n",
" factory = HubDatasetModuleFactoryWithScript(task,trust_remote_code=False)\n",
" print(factory.get_module())\n",
" except Exception as e:\n",
" if \"trust_remote_code=True\" in str(e):\n",
" print(str(e))\n",
" trust_remote_code_tasks.append(factory.name)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "2022c5db-a123-4691-80b9-a6cd1c25a4c3",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['EleutherAI/mutual',\n",
" 'skt/kobest_v1',\n",
" 'EleutherAI/logiqa',\n",
" 'bigbio/pubmed_qa',\n",
" 'EleutherAI/race',\n",
" 'BigScienceBiasEval/crows_pairs_multilingual',\n",
" 'baber/logiqa2',\n",
" 'EleutherAI/arithmetic',\n",
" 'EleutherAI/asdiv',\n",
" 'EleutherAI/lambada_openai',\n",
" 'EleutherAI/wikitext_document_level',\n",
" 'EleutherAI/drop',\n",
" 'EleutherAI/hendrycks_math',\n",
" 'juletxara/xstory_cloze',\n",
" 'allenai/qasper',\n",
" 'EleutherAI/pile',\n",
" 'EleutherAI/sycophancy',\n",
" 'EleutherAI/hendrycks_ethics',\n",
" 'EleutherAI/unscramble',\n",
" 'EleutherAI/headqa',\n",
" 'skg/toxigen-data',\n",
" 'EleutherAI/coqa',\n",
" 'corypaik/prost']"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"trust_remote_code_tasks"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.8"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment