Qwen3 JMTEB config

qwen3_embedding_jmteb.jsonnet

// ========== amazon_counterfactual_classification.jsonnet ==========
{
  amazon_counterfactual_classification: {
    class_path: 'ClassificationEvaluator',
    init_args: {
      train_dataset: {
        class_path: 'HfClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'train',
          name: 'amazon_counterfactual_classification',
        },
      },
      val_dataset: {
        class_path: 'HfClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'amazon_counterfactual_classification',
        },
      },
      test_dataset: {
        class_path: 'HfClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'amazon_counterfactual_classification',
        },
      },
    },
  },
}

// ========== amazon_review_classification.jsonnet ==========
{
  amazon_review_classification: {
    class_path: 'ClassificationEvaluator',
    init_args: {
      train_dataset: {
        class_path: 'HfClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'train',
          name: 'amazon_review_classification',
        },
      },
      val_dataset: {
        class_path: 'HfClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'amazon_review_classification',
        },
      },
      test_dataset: {
        class_path: 'HfClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'amazon_review_classification',
        },
      },
    },
  },
}

// ========== esci.jsonnet ==========
{
  esci: {
    class_path: 'RerankingEvaluator',
    init_args: {
      query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
      val_query_dataset: {
        class_path: 'HfRerankingQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'esci-query',
        },
      },
      test_query_dataset: {
        class_path: 'HfRerankingQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'esci-query',
        },
      },
      doc_dataset: {
        class_path: 'HfRerankingDocDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'corpus',
          name: 'esci-corpus',
        },
      },
    },
  },
}

// ========== jagovfaqs_22k.jsonnet ==========
{
  jagovfaqs_22k: {
    class_path: 'RetrievalEvaluator',
    init_args: {
      query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
      doc_prefix: '',
      val_query_dataset: {
        class_path: 'HfRetrievalQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'jagovfaqs_22k-query',
        },
      },
      test_query_dataset: {
        class_path: 'HfRetrievalQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'jagovfaqs_22k-query',
        },
      },
      doc_dataset: {
        class_path: 'HfRetrievalDocDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'corpus',
          name: 'jagovfaqs_22k-corpus',
        },
      },
    },
  },
}

// ========== jaqket.jsonnet ==========
{
  jaqket: {
    class_path: 'RetrievalEvaluator',
    init_args: {
      query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
      val_query_dataset: {
        class_path: 'HfRetrievalQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'jaqket-query',
        },
      },
      test_query_dataset: {
        class_path: 'HfRetrievalQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'jaqket-query',
        },
      },
      doc_dataset: {
        class_path: 'HfRetrievalDocDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'corpus',
          name: 'jaqket-corpus',
        },
      },
    },
  },
}

// ========== jsick.jsonnet ==========
{
  jsick: {
    class_path: 'STSEvaluator',
    init_args: {
      val_dataset: {
        class_path: 'HfSTSDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'jsick',
        },
      },
      test_dataset: {
        class_path: 'HfSTSDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'jsick',
        },
      },
    },
  },
}

// ========== jsts.jsonnet ==========
{
  jsts: {
    class_path: 'STSEvaluator',
    init_args: {
      val_dataset: {
        class_path: 'HfSTSDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'train',
          name: 'jsts',
        },
      },
      test_dataset: {
        class_path: 'HfSTSDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'jsts',
        },
      },
    },
  },
}

// ========== livedoor_news.jsonnet ==========
{
  livedoor_news: {
    class_path: 'ClusteringEvaluator',
    init_args: {
      val_dataset: {
        class_path: 'HfClusteringDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'livedoor_news',
        },
      },
      test_dataset: {
        class_path: 'HfClusteringDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'livedoor_news',
        },
      },
    },
  },
}

// ========== massive_intent_classification.jsonnet ==========
{
  massive_intent_classification: {
    class_path: 'ClassificationEvaluator',
    init_args: {
      train_dataset: {
        class_path: 'HfClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'train',
          name: 'massive_intent_classification',
        },
      },
      val_dataset: {
        class_path: 'HfClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'massive_intent_classification',
        },
      },
      test_dataset: {
        class_path: 'HfClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'massive_intent_classification',
        },
      },
    },
  },
}

// ========== massive_scenario_classification.jsonnet ==========
{
  massive_scenario_classification: {
    class_path: 'ClassificationEvaluator',
    init_args: {
      train_dataset: {
        class_path: 'HfClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'train',
          name: 'massive_scenario_classification',
        },
      },
      val_dataset: {
        class_path: 'HfClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'massive_scenario_classification',
        },
      },
      test_dataset: {
        class_path: 'HfClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'massive_scenario_classification',
        },
      },
    },
  },
}

// ========== mewsc16.jsonnet ==========
{
  mewsc16: {
    class_path: 'ClusteringEvaluator',
    init_args: {
      val_dataset: {
        class_path: 'HfClusteringDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'mewsc16_ja',
        },
      },
      test_dataset: {
        class_path: 'HfClusteringDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'mewsc16_ja',
        },
      },
    },
  },
}

// ========== mrtydi.jsonnet ==========
{
  mrtydi: {
    class_path: 'RetrievalEvaluator',
    init_args: {
      query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
      val_query_dataset: {
        class_path: 'HfRetrievalQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'mrtydi-query',
        },
      },
      test_query_dataset: {
        class_path: 'HfRetrievalQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'mrtydi-query',
        },
      },
      doc_dataset: {
        class_path: 'HfRetrievalDocDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'corpus',
          name: 'mrtydi-corpus',
        },
      },
    },
  },
}

// ========== nlp_journal_abs_intro.jsonnet ==========
{
  nlp_journal_abs_intro: {
    class_path: 'RetrievalEvaluator',
    init_args: {
      query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
      val_query_dataset: {
        class_path: 'HfRetrievalQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'nlp_journal_abs_intro-query',
        },
      },
      test_query_dataset: {
        class_path: 'HfRetrievalQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'nlp_journal_abs_intro-query',
        },
      },
      doc_dataset: {
        class_path: 'HfRetrievalDocDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'corpus',
          name: 'nlp_journal_abs_intro-corpus',
        },
      },
    },
  },
}

// ========== nlp_journal_title_abs.jsonnet ==========
{
  nlp_journal_title_abs: {
    class_path: 'RetrievalEvaluator',
    init_args: {
      query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
      val_query_dataset: {
        class_path: 'HfRetrievalQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'nlp_journal_title_abs-query',
        },
      },
      test_query_dataset: {
        class_path: 'HfRetrievalQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'nlp_journal_title_abs-query',
        },
      },
      doc_dataset: {
        class_path: 'HfRetrievalDocDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'corpus',
          name: 'nlp_journal_title_abs-corpus',
        },
      },
    },
  },
}

// ========== nlp_journal_title_intro.jsonnet ==========
{
  nlp_journal_title_intro: {
    class_path: 'RetrievalEvaluator',
    init_args: {
      query_prefix: 'Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery:',
      val_query_dataset: {
        class_path: 'HfRetrievalQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'nlp_journal_title_intro-query',
        },
      },
      test_query_dataset: {
        class_path: 'HfRetrievalQueryDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'nlp_journal_title_intro-query',
        },
      },
      doc_dataset: {
        class_path: 'HfRetrievalDocDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'corpus',
          name: 'nlp_journal_title_intro-corpus',
        },
      },
    },
  },
}

// ========== paws_x_ja.jsonnet ==========
{
  paws_x_ja: {
    class_path: 'PairClassificationEvaluator',
    init_args: {
      val_dataset: {
        class_path: 'HfPairClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'validation',
          name: 'paws_x_ja',
        },
      },
      test_dataset: {
        class_path: 'HfPairClassificationDataset',
        init_args: {
          path: 'sbintuitions/JMTEB',
          split: 'test',
          name: 'paws_x_ja',
        },
      },
    },
  },
}

summary.json

{
    "Classification": {
        "amazon_counterfactual_classification": {
            "macro_f1": 0.61547962124331
        },
        "amazon_review_classification": {
            "macro_f1": 0.3697166662303661
        },
        "massive_intent_classification": {
            "macro_f1": 0.7748579959778433
        },
        "massive_scenario_classification": {
            "macro_f1": 0.883560637303876
        }
    },
    "Reranking": {
        "esci": {
            "ndcg@10": 0.9310073173693335
        }
    },
    "Retrieval": {
        "jagovfaqs_22k": {
            "ndcg@10": 0.659288467153427
        },
        "jaqket": {
            "ndcg@10": 0.4415379744454133
        },
        "mrtydi": {
            "ndcg@10": 0.3946393778964397
        },
        "nlp_journal_abs_intro": {
            "ndcg@10": 0.9811631239095655
        },
        "nlp_journal_title_abs": {
            "ndcg@10": 0.9588062326881761
        },
        "nlp_journal_title_intro": {
            "ndcg@10": 0.9330314380838046
        }
    },
    "Clustering": {
        "livedoor_news": {
            "v_measure_score": 0.508413289574316
        },
        "mewsc16": {
            "v_measure_score": 0.46844428287466244
        }
    },
    "PairClassification": {
        "paws_x_ja": {
            "binary_f1": 0.6242038216560509
        }
    }
}

以下で実行

poetry run python -m jmteb \
  --embedder SentenceBertEmbedder \
  --embedder.model_name_or_path 'Qwen/Qwen3-Embedding-0.6B' \
  --embedder.model_kwargs '{"torch_dtype": "torch.bfloat16"}' \
  --embedder.device cuda \
  --embedder.batch_size 32 \
  --embedder.max_seq_length 2048 \
  --evaluators 'jmteb_config/qwen3_embedding_jmteb.jsonnet' \
  --save_dir 'results/Qwen_Qwen3-Embedding-0.6B' \
  --overwrite_cache false