Skip to content

Instantly share code, notes, and snippets.

@hotchpotch

hotchpotch/qwen3_embedding_jmteb.jsonnet Secret

Last active June 10, 2025 21:28
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save hotchpotch/f6be186010e70d6eb6e46447cea258f9 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save hotchpotch/f6be186010e70d6eb6e46447cea258f9 to your computer and use it in GitHub Desktop.
Download ZIP
Qwen3 JMTEB config
Raw
qwen3_embedding_jmteb.jsonnet
// ========== amazon_counterfactual_classification.jsonnet ==========
{
amazon_counterfactual_classification: {
class_path: 'ClassificationEvaluator',
init_args: {
train_dataset: {
class_path: 'HfClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'train',
name: 'amazon_counterfactual_classification',
},
},
val_dataset: {
class_path: 'HfClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'amazon_counterfactual_classification',
},
},
test_dataset: {
class_path: 'HfClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'amazon_counterfactual_classification',
},
},
},
},
}
// ========== amazon_review_classification.jsonnet ==========
{
amazon_review_classification: {
class_path: 'ClassificationEvaluator',
init_args: {
train_dataset: {
class_path: 'HfClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'train',
name: 'amazon_review_classification',
},
},
val_dataset: {
class_path: 'HfClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'amazon_review_classification',
},
},
test_dataset: {
class_path: 'HfClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'amazon_review_classification',
},
},
},
},
}
// ========== esci.jsonnet ==========
{
esci: {
class_path: 'RerankingEvaluator',
init_args: {
query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
val_query_dataset: {
class_path: 'HfRerankingQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'esci-query',
},
},
test_query_dataset: {
class_path: 'HfRerankingQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'esci-query',
},
},
doc_dataset: {
class_path: 'HfRerankingDocDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'corpus',
name: 'esci-corpus',
},
},
},
},
}
// ========== jagovfaqs_22k.jsonnet ==========
{
jagovfaqs_22k: {
class_path: 'RetrievalEvaluator',
init_args: {
query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
doc_prefix: '',
val_query_dataset: {
class_path: 'HfRetrievalQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'jagovfaqs_22k-query',
},
},
test_query_dataset: {
class_path: 'HfRetrievalQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'jagovfaqs_22k-query',
},
},
doc_dataset: {
class_path: 'HfRetrievalDocDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'corpus',
name: 'jagovfaqs_22k-corpus',
},
},
},
},
}
// ========== jaqket.jsonnet ==========
{
jaqket: {
class_path: 'RetrievalEvaluator',
init_args: {
query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
val_query_dataset: {
class_path: 'HfRetrievalQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'jaqket-query',
},
},
test_query_dataset: {
class_path: 'HfRetrievalQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'jaqket-query',
},
},
doc_dataset: {
class_path: 'HfRetrievalDocDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'corpus',
name: 'jaqket-corpus',
},
},
},
},
}
// ========== jsick.jsonnet ==========
{
jsick: {
class_path: 'STSEvaluator',
init_args: {
val_dataset: {
class_path: 'HfSTSDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'jsick',
},
},
test_dataset: {
class_path: 'HfSTSDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'jsick',
},
},
},
},
}
// ========== jsts.jsonnet ==========
{
jsts: {
class_path: 'STSEvaluator',
init_args: {
val_dataset: {
class_path: 'HfSTSDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'train',
name: 'jsts',
},
},
test_dataset: {
class_path: 'HfSTSDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'jsts',
},
},
},
},
}
// ========== livedoor_news.jsonnet ==========
{
livedoor_news: {
class_path: 'ClusteringEvaluator',
init_args: {
val_dataset: {
class_path: 'HfClusteringDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'livedoor_news',
},
},
test_dataset: {
class_path: 'HfClusteringDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'livedoor_news',
},
},
},
},
}
// ========== massive_intent_classification.jsonnet ==========
{
massive_intent_classification: {
class_path: 'ClassificationEvaluator',
init_args: {
train_dataset: {
class_path: 'HfClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'train',
name: 'massive_intent_classification',
},
},
val_dataset: {
class_path: 'HfClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'massive_intent_classification',
},
},
test_dataset: {
class_path: 'HfClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'massive_intent_classification',
},
},
},
},
}
// ========== massive_scenario_classification.jsonnet ==========
{
massive_scenario_classification: {
class_path: 'ClassificationEvaluator',
init_args: {
train_dataset: {
class_path: 'HfClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'train',
name: 'massive_scenario_classification',
},
},
val_dataset: {
class_path: 'HfClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'massive_scenario_classification',
},
},
test_dataset: {
class_path: 'HfClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'massive_scenario_classification',
},
},
},
},
}
// ========== mewsc16.jsonnet ==========
{
mewsc16: {
class_path: 'ClusteringEvaluator',
init_args: {
val_dataset: {
class_path: 'HfClusteringDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'mewsc16_ja',
},
},
test_dataset: {
class_path: 'HfClusteringDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'mewsc16_ja',
},
},
},
},
}
// ========== mrtydi.jsonnet ==========
{
mrtydi: {
class_path: 'RetrievalEvaluator',
init_args: {
query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
val_query_dataset: {
class_path: 'HfRetrievalQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'mrtydi-query',
},
},
test_query_dataset: {
class_path: 'HfRetrievalQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'mrtydi-query',
},
},
doc_dataset: {
class_path: 'HfRetrievalDocDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'corpus',
name: 'mrtydi-corpus',
},
},
},
},
}
// ========== nlp_journal_abs_intro.jsonnet ==========
{
nlp_journal_abs_intro: {
class_path: 'RetrievalEvaluator',
init_args: {
query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
val_query_dataset: {
class_path: 'HfRetrievalQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'nlp_journal_abs_intro-query',
},
},
test_query_dataset: {
class_path: 'HfRetrievalQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'nlp_journal_abs_intro-query',
},
},
doc_dataset: {
class_path: 'HfRetrievalDocDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'corpus',
name: 'nlp_journal_abs_intro-corpus',
},
},
},
},
}
// ========== nlp_journal_title_abs.jsonnet ==========
{
nlp_journal_title_abs: {
class_path: 'RetrievalEvaluator',
init_args: {
query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
val_query_dataset: {
class_path: 'HfRetrievalQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'nlp_journal_title_abs-query',
},
},
test_query_dataset: {
class_path: 'HfRetrievalQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'nlp_journal_title_abs-query',
},
},
doc_dataset: {
class_path: 'HfRetrievalDocDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'corpus',
name: 'nlp_journal_title_abs-corpus',
},
},
},
},
}
// ========== nlp_journal_title_intro.jsonnet ==========
{
nlp_journal_title_intro: {
class_path: 'RetrievalEvaluator',
init_args: {
query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
val_query_dataset: {
class_path: 'HfRetrievalQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'nlp_journal_title_intro-query',
},
},
test_query_dataset: {
class_path: 'HfRetrievalQueryDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'nlp_journal_title_intro-query',
},
},
doc_dataset: {
class_path: 'HfRetrievalDocDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'corpus',
name: 'nlp_journal_title_intro-corpus',
},
},
},
},
}
// ========== paws_x_ja.jsonnet ==========
{
paws_x_ja: {
class_path: 'PairClassificationEvaluator',
init_args: {
val_dataset: {
class_path: 'HfPairClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'validation',
name: 'paws_x_ja',
},
},
test_dataset: {
class_path: 'HfPairClassificationDataset',
init_args: {
path: 'sbintuitions/JMTEB',
split: 'test',
name: 'paws_x_ja',
},
},
},
},
}
Raw
summary.json
{
"Classification": {
"amazon_counterfactual_classification": {
"macro_f1": 0.61547962124331
},
"amazon_review_classification": {
"macro_f1": 0.3697166662303661
},
"massive_intent_classification": {
"macro_f1": 0.7748579959778433
},
"massive_scenario_classification": {
"macro_f1": 0.883560637303876
}
},
"Reranking": {
"esci": {
"ndcg@10": 0.9310073173693335
}
},
"Retrieval": {
"jagovfaqs_22k": {
"ndcg@10": 0.659288467153427
},
"jaqket": {
"ndcg@10": 0.4415379744454133
},
"mrtydi": {
"ndcg@10": 0.3946393778964397
},
"nlp_journal_abs_intro": {
"ndcg@10": 0.9811631239095655
},
"nlp_journal_title_abs": {
"ndcg@10": 0.9588062326881761
},
"nlp_journal_title_intro": {
"ndcg@10": 0.9330314380838046
}
},
"Clustering": {
"livedoor_news": {
"v_measure_score": 0.508413289574316
},
"mewsc16": {
"v_measure_score": 0.46844428287466244
}
},
"PairClassification": {
"paws_x_ja": {
"binary_f1": 0.6242038216560509
}
}
}
@hotchpotch
Copy link
Author

hotchpotch commented Jun 10, 2025

以下で実行

poetry run python -m jmteb \
  --embedder SentenceBertEmbedder \
  --embedder.model_name_or_path 'Qwen/Qwen3-Embedding-0.6B' \
  --embedder.model_kwargs '{"torch_dtype": "torch.bfloat16"}' \
  --embedder.device cuda \
  --embedder.batch_size 32 \
  --embedder.max_seq_length 2048 \
  --evaluators 'jmteb_config/qwen3_embedding_jmteb.jsonnet' \
  --save_dir 'results/Qwen_Qwen3-Embedding-0.6B' \
  --overwrite_cache false

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment