MetaR reproduced results - config = (Pretrain/Intrain, # of runs, batch size, # of shots])

MetaR results

{"("Pre-Train", 10, 64, 1)": {
    "MRR_ci": [0.189, 0.212, 0.225], 
    "Hit@10_ci": [0.283, 0.331, 0.351], 
    "Hit@5_ci": [0.227, 0.268, 0.283], 
    "Hit@1_ci": [0.137, 0.149, 0.163], 
    "MRR": [0.208, 0.217, 0.204, 0.212, 0.186, 0.199, 0.227, 0.211, 0.217, 0.212], 
    "Hit@10": [0.343, 0.329, 0.312, 0.339, 0.276, 0.308, 0.344, 0.329, 0.333, 0.353], 
    "Hit@5": [0.264, 0.273, 0.253, 0.271, 0.22, 0.262, 0.286, 0.265, 0.273, 0.271], 
    "Hit@1": [0.149, 0.158, 0.147, 0.149, 0.141, 0.136, 0.165, 0.15, 0.155, 0.148]
}, 
"("Pre-Train", 10, 64, 5)": {
    "MRR_ci": [0.184, 0.208, 0.219], 
    "Hit@10_ci": [0.319, 0.348, 0.358], 
    "Hit@5_ci": [0.244, 0.276, 0.302], 
    "Hit@1_ci": [0.106, 0.14, 0.15], 
    "MRR": [0.183, 0.219, 0.208, 0.196, 0.193, 0.211, 0.207, 0.217, 0.209, 0.189], 
    "Hit@10": [0.317, 0.359, 0.348, 0.324, 0.354, 0.332, 0.356, 0.348, 0.33, 0.347], 
    "Hit@5": [0.239, 0.3, 0.303, 0.274, 0.3, 0.259, 0.264, 0.28, 0.272, 0.277], 
    "Hit@1": [0.118, 0.144, 0.135, 0.122, 0.103, 0.15, 0.144, 0.149, 0.146, 0.115]
},
"("Pre-Train", 10, 1024, 1)": {
    "MRR_ci": [0.212, 0.22, 0.229],
    "Hit@10_ci": [0.323, 0.344, 0.378],
    "Hit@5_ci": [0.262, 0.286, 0.311],
    "Hit@1_ci": [0.132, 0.156, 0.163],
    "MRR": [0.224, 0.215, 0.22, 0.214, 0.222, 0.213, 0.212, 0.224, 0.23, 0.221],
    "Hit@10": [0.356, 0.323, 0.345, 0.384, 0.329, 0.334, 0.323, 0.349, 0.342, 0.349],
    "Hit@5": [0.293, 0.26, 0.281, 0.315, 0.282, 0.268, 0.273, 0.297, 0.296, 0.29],
    "Hit@1": [0.156, 0.159, 0.156, 0.127, 0.161, 0.151, 0.151, 0.153, 0.164, 0.155]
},
"("Pre-Train", 10, 1024, 5)": {
    "MRR_ci": [0.177, 0.213, 0.234],
    "Hit@10_ci": [0.292, 0.334, 0.379],
    "Hit@5_ci": [0.234, 0.278, 0.32],
    "Hit@1_ci": [0.116, 0.142, 0.154],
    "MRR": [0.17, 0.203, 0.223, 0.206, 0.215, 0.207, 0.235, 0.229, 0.211, 0.218],
    "Hit@10": [0.282, 0.33, 0.341, 0.325, 0.33, 0.331, 0.372, 0.375, 0.337, 0.38],
    "Hit@5": [0.224, 0.268, 0.285, 0.272, 0.274, 0.283, 0.311, 0.322, 0.274, 0.304],
    "Hit@1": [0.112, 0.136, 0.15, 0.136, 0.148, 0.131, 0.154, 0.153, 0.147, 0.13]
},
"("In-Train", 10, 64, 1)": {
    "MRR_ci": [0.126, 0.216, 0.239],
    "Hit@10_ci": [0.227, 0.36, 0.386],
    "Hit@5_ci": [0.16, 0.289, 0.32],
    "Hit@1_ci": [0.074, 0.137, 0.156],
    "MRR": [0.222, 0.116, 0.212, 0.188, 0.183, 0.236, 0.235, 0.24, 0.22, 0.159],
    "Hit@10": [0.384, 0.213, 0.356, 0.303, 0.289, 0.384, 0.37, 0.386, 0.363, 0.276],
    "Hit@5": [0.315, 0.143, 0.284, 0.245, 0.236, 0.318, 0.319, 0.32, 0.294, 0.219],
    "Hit@1": [0.14, 0.067, 0.134, 0.122, 0.129, 0.156, 0.156, 0.156, 0.141, 0.099]
},
"("In-Train", 10, 64, 5)": {
    "MRR_ci": [0.1, 0.206, 0.265],
    "Hit@10_ci": [0.179, 0.379, 0.463],
    "Hit@5_ci": [0.131, 0.281, 0.37],
    "Hit@1_ci": [0.051, 0.116, 0.165],
    "MRR": [0.187, 0.228, 0.094, 0.175, 0.252, 0.224, 0.134, 0.119, 0.269, 0.243],
    "Hit@10": [0.355, 0.403, 0.165, 0.309, 0.434, 0.417, 0.229, 0.266, 0.472, 0.417],
    "Hit@5": [0.267, 0.295, 0.117, 0.229, 0.348, 0.326, 0.18, 0.185, 0.377, 0.317],
    "Hit@1": [0.106, 0.142, 0.055, 0.103, 0.163, 0.127, 0.074, 0.05, 0.165, 0.157]
},
"("In-Train", 10, 1024, 1)": {
    "MRR_ci": [0.185, 0.244, 0.288],
    "Hit@10_ci": [0.32, 0.407, 0.447],
    "Hit@5_ci": [0.246, 0.326, 0.382],
    "Hit@1_ci": [0.111, 0.164, 0.2],
    "MRR": [0.29, 0.195, 0.226, 0.231, 0.256, 0.282, 0.183, 0.19, 0.283, 0.268],
    "Hit@10": [0.444, 0.321, 0.377, 0.39, 0.424, 0.448, 0.324, 0.32, 0.444, 0.429],
    "Hit@5": [0.384, 0.251, 0.284, 0.306, 0.348, 0.374, 0.264, 0.244, 0.375, 0.347],
    "Hit@1": [0.202, 0.133, 0.156, 0.15, 0.171, 0.194, 0.107, 0.124, 0.195, 0.186]
}}