-
-
Save simonw/73d15c0dd1025d1196829740bacf4464 to your computer and use it in GitHub Desktop.
Sizes of files in https://data.together.xyz/redpajama-data-1T/v1.0.0/urls.txt - Datasette Lite demo: https://lite.datasette.io/?json=https://gist.github.com/simonw/73d15c0dd1025d1196829740bacf4464
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [ | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2149112128, | |
| "size_mb": 2049.5530395507812, | |
| "size_gb": 2.00151664018631, | |
| "path": "common_crawl/2019-30/en_head_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1390156197, | |
| "size_mb": 1325.7562608718872, | |
| "size_gb": 1.2946838485077024, | |
| "path": "common_crawl/2019-30/en_middle_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2209631095, | |
| "size_mb": 2107.2684240341187, | |
| "size_gb": 2.057879320345819, | |
| "path": "common_crawl/2023-06/en_head_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2192667877, | |
| "size_mb": 2091.0910387039185, | |
| "size_gb": 2.0420810924842954, | |
| "path": "common_crawl/2019-30/en_head_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1403746107, | |
| "size_mb": 1338.7166090011597, | |
| "size_gb": 1.307340438477695, | |
| "path": "common_crawl/2019-30/en_middle_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1424666886, | |
| "size_mb": 1358.668218612671, | |
| "size_gb": 1.3268244322389364, | |
| "path": "common_crawl/2019-30/en_middle_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2221582051, | |
| "size_mb": 2118.66574382782, | |
| "size_gb": 2.0690095154568553, | |
| "path": "common_crawl/2023-06/en_head_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2136901093, | |
| "size_mb": 2037.9076890945435, | |
| "size_gb": 1.99014422763139, | |
| "path": "common_crawl/2019-30/en_head_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2131365580, | |
| "size_mb": 2032.6286125183105, | |
| "size_gb": 1.9849888794124126, | |
| "path": "common_crawl/2019-30/en_head_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2186818891, | |
| "size_mb": 2085.5130109786987, | |
| "size_gb": 2.0366337997838855, | |
| "path": "common_crawl/2023-06/en_head_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2178831270, | |
| "size_mb": 2077.8954219818115, | |
| "size_gb": 2.029194748029113, | |
| "path": "common_crawl/2023-06/en_head_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2207792663, | |
| "size_mb": 2105.5151586532593, | |
| "size_gb": 2.0561671471223235, | |
| "path": "common_crawl/2023-06/en_head_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2161485547, | |
| "size_mb": 2061.3532514572144, | |
| "size_gb": 2.013040284626186, | |
| "path": "common_crawl/2019-30/en_head_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2096959805, | |
| "size_mb": 1999.8167085647583, | |
| "size_gb": 1.9529460044577718, | |
| "path": "common_crawl/2023-06/en_head_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2190504212, | |
| "size_mb": 2089.0276069641113, | |
| "size_gb": 2.04006602242589, | |
| "path": "common_crawl/2023-06/en_head_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1436336586, | |
| "size_mb": 1369.797311782837, | |
| "size_gb": 1.3376926872879267, | |
| "path": "common_crawl/2019-30/en_middle_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2140313740, | |
| "size_mb": 2041.1622428894043, | |
| "size_gb": 1.9933225028216839, | |
| "path": "common_crawl/2019-30/en_head_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2187351650, | |
| "size_mb": 2086.021089553833, | |
| "size_gb": 2.037129970267415, | |
| "path": "common_crawl/2023-06/en_head_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2234393065, | |
| "size_mb": 2130.8832788467407, | |
| "size_gb": 2.0809407019987702, | |
| "path": "common_crawl/2023-06/en_head_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2183791958, | |
| "size_mb": 2082.626302719116, | |
| "size_gb": 2.033814748749137, | |
| "path": "common_crawl/2023-06/en_head_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1401349568, | |
| "size_mb": 1336.4310913085938, | |
| "size_gb": 1.3051084876060486, | |
| "path": "common_crawl/2019-30/en_middle_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2263107623, | |
| "size_mb": 2158.2676153182983, | |
| "size_gb": 2.1076832180842757, | |
| "path": "common_crawl/2023-06/en_head_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2213195975, | |
| "size_mb": 2110.668158531189, | |
| "size_gb": 2.0611993735656142, | |
| "path": "common_crawl/2023-06/en_head_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2185729366, | |
| "size_mb": 2084.473958969116, | |
| "size_gb": 2.0356191005557775, | |
| "path": "common_crawl/2023-06/en_head_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1421734518, | |
| "size_mb": 1355.8716945648193, | |
| "size_gb": 1.3240934517234564, | |
| "path": "common_crawl/2019-30/en_middle_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2169765338, | |
| "size_mb": 2069.249475479126, | |
| "size_gb": 2.020751440897584, | |
| "path": "common_crawl/2019-30/en_head_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2063686084, | |
| "size_mb": 1968.084415435791, | |
| "size_gb": 1.9219574369490147, | |
| "path": "common_crawl/2023-06/en_head_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2206732391, | |
| "size_mb": 2104.5040044784546, | |
| "size_gb": 2.055179691873491, | |
| "path": "common_crawl/2019-30/en_head_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1444685375, | |
| "size_mb": 1377.759337425232, | |
| "size_gb": 1.345468102954328, | |
| "path": "common_crawl/2019-30/en_middle_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2175747864, | |
| "size_mb": 2074.9548568725586, | |
| "size_gb": 2.026323102414608, | |
| "path": "common_crawl/2023-06/en_head_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2251297614, | |
| "size_mb": 2147.0047130584717, | |
| "size_gb": 2.0966842900961637, | |
| "path": "common_crawl/2023-06/en_head_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2281644532, | |
| "size_mb": 2175.945789337158, | |
| "size_gb": 2.1249470598995686, | |
| "path": "common_crawl/2023-06/en_head_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2206388430, | |
| "size_mb": 2104.175977706909, | |
| "size_gb": 2.0548593532294035, | |
| "path": "common_crawl/2019-30/en_head_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1423990242, | |
| "size_mb": 1358.0229206085205, | |
| "size_gb": 1.3261942584067583, | |
| "path": "common_crawl/2019-30/en_middle_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2240596535, | |
| "size_mb": 2136.7993688583374, | |
| "size_gb": 2.08671813365072, | |
| "path": "common_crawl/2023-06/en_head_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2187852891, | |
| "size_mb": 2086.499110221863, | |
| "size_gb": 2.037596787326038, | |
| "path": "common_crawl/2023-06/en_head_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2097685422, | |
| "size_mb": 2000.508710861206, | |
| "size_gb": 1.9536217879503965, | |
| "path": "common_crawl/2019-30/en_head_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2206035243, | |
| "size_mb": 2103.8391523361206, | |
| "size_gb": 2.054530422203243, | |
| "path": "common_crawl/2023-06/en_head_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1407669996, | |
| "size_mb": 1342.4587211608887, | |
| "size_gb": 1.3109948448836803, | |
| "path": "common_crawl/2019-30/en_middle_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1456685960, | |
| "size_mb": 1389.203987121582, | |
| "size_gb": 1.35664451867342, | |
| "path": "common_crawl/2019-30/en_middle_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2233567920, | |
| "size_mb": 2130.0963592529297, | |
| "size_gb": 2.080172225832939, | |
| "path": "common_crawl/2023-06/en_head_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2126895282, | |
| "size_mb": 2028.3654041290283, | |
| "size_gb": 1.9808255899697542, | |
| "path": "common_crawl/2023-06/en_head_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2173337745, | |
| "size_mb": 2072.656388282776, | |
| "size_gb": 2.0240785041823983, | |
| "path": "common_crawl/2019-30/en_head_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2208731121, | |
| "size_mb": 2106.4101419448853, | |
| "size_gb": 2.057041154243052, | |
| "path": "common_crawl/2023-06/en_head_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2203373238, | |
| "size_mb": 2101.3004665374756, | |
| "size_gb": 2.0520512368530035, | |
| "path": "common_crawl/2023-06/en_head_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2153855773, | |
| "size_mb": 2054.07693195343, | |
| "size_gb": 2.0059345038607717, | |
| "path": "common_crawl/2019-30/en_head_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1416111792, | |
| "size_mb": 1350.5094451904297, | |
| "size_gb": 1.318856880068779, | |
| "path": "common_crawl/2019-30/en_middle_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2196006895, | |
| "size_mb": 2094.2753744125366, | |
| "size_gb": 2.045190795324743, | |
| "path": "common_crawl/2019-30/en_head_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2100249614, | |
| "size_mb": 2002.9541149139404, | |
| "size_gb": 1.956009877845645, | |
| "path": "common_crawl/2019-30/en_head_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2189223281, | |
| "size_mb": 2087.8060159683228, | |
| "size_gb": 2.038873062469065, | |
| "path": "common_crawl/2023-06/en_head_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1399627138, | |
| "size_mb": 1334.7884540557861, | |
| "size_gb": 1.3035043496638536, | |
| "path": "common_crawl/2019-30/en_middle_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1426475209, | |
| "size_mb": 1360.3927698135376, | |
| "size_gb": 1.3285085642710328, | |
| "path": "common_crawl/2019-30/en_middle_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2174830587, | |
| "size_mb": 2074.0800733566284, | |
| "size_gb": 2.0254688216373324, | |
| "path": "common_crawl/2023-06/en_head_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1410955529, | |
| "size_mb": 1345.5920495986938, | |
| "size_gb": 1.3140547359362245, | |
| "path": "common_crawl/2019-30/en_middle_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1379469494, | |
| "size_mb": 1315.5646266937256, | |
| "size_gb": 1.2847310807555914, | |
| "path": "common_crawl/2019-30/en_middle_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2297774634, | |
| "size_mb": 2191.3286533355713, | |
| "size_gb": 2.139969388023019, | |
| "path": "common_crawl/2023-06/en_head_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1395368648, | |
| "size_mb": 1330.7272415161133, | |
| "size_gb": 1.2995383217930794, | |
| "path": "common_crawl/2019-30/en_middle_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2226013740, | |
| "size_mb": 2122.89213180542, | |
| "size_gb": 2.0731368474662304, | |
| "path": "common_crawl/2023-06/en_head_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2195061859, | |
| "size_mb": 2093.3741178512573, | |
| "size_gb": 2.0443106619641185, | |
| "path": "common_crawl/2019-30/en_head_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2145364782, | |
| "size_mb": 2045.9792919158936, | |
| "size_gb": 1.9980266522616148, | |
| "path": "common_crawl/2019-30/en_head_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2231171652, | |
| "size_mb": 2127.8111000061035, | |
| "size_gb": 2.0779405273497105, | |
| "path": "common_crawl/2023-06/en_head_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2219482503, | |
| "size_mb": 2116.6634588241577, | |
| "size_gb": 2.0670541590079665, | |
| "path": "common_crawl/2023-06/en_head_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2203176132, | |
| "size_mb": 2101.112491607666, | |
| "size_gb": 2.0518676675856113, | |
| "path": "common_crawl/2023-06/en_head_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2131056157, | |
| "size_mb": 2032.3335237503052, | |
| "size_gb": 1.9847007067874074, | |
| "path": "common_crawl/2019-30/en_head_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2229201967, | |
| "size_mb": 2125.932662010193, | |
| "size_gb": 2.076106115244329, | |
| "path": "common_crawl/2019-30/en_head_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2219557462, | |
| "size_mb": 2116.734945297241, | |
| "size_gb": 2.067123970016837, | |
| "path": "common_crawl/2019-30/en_head_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2217678267, | |
| "size_mb": 2114.942805290222, | |
| "size_gb": 2.0653738332912326, | |
| "path": "common_crawl/2019-30/en_head_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2152398527, | |
| "size_mb": 2052.6871938705444, | |
| "size_gb": 2.0045773377642035, | |
| "path": "common_crawl/2019-30/en_head_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2246726594, | |
| "size_mb": 2142.6454486846924, | |
| "size_gb": 2.092427195981145, | |
| "path": "common_crawl/2023-06/en_head_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2207886694, | |
| "size_mb": 2105.6048336029053, | |
| "size_gb": 2.056254720315337, | |
| "path": "common_crawl/2023-06/en_head_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2217902163, | |
| "size_mb": 2115.1563291549683, | |
| "size_gb": 2.0655823526903987, | |
| "path": "common_crawl/2023-06/en_head_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1226100799, | |
| "size_mb": 1169.300841331482, | |
| "size_gb": 1.1418953528627753, | |
| "path": "common_crawl/2019-30/en_head_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2255699790, | |
| "size_mb": 2151.2029552459717, | |
| "size_gb": 2.100784135982394, | |
| "path": "common_crawl/2023-06/en_head_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2237187626, | |
| "size_mb": 2133.5483798980713, | |
| "size_gb": 2.0835433397442102, | |
| "path": "common_crawl/2023-06/en_head_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2201724735, | |
| "size_mb": 2099.728331565857, | |
| "size_gb": 2.050515948794782, | |
| "path": "common_crawl/2023-06/en_head_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2166917873, | |
| "size_mb": 2066.5339212417603, | |
| "size_gb": 2.0180995324626565, | |
| "path": "common_crawl/2019-30/en_head_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2183337458, | |
| "size_mb": 2082.1928577423096, | |
| "size_gb": 2.033391462638974, | |
| "path": "common_crawl/2023-06/en_head_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1424697633, | |
| "size_mb": 1358.6975412368774, | |
| "size_gb": 1.3268530676141381, | |
| "path": "common_crawl/2019-30/en_middle_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1386605095, | |
| "size_mb": 1322.3696660995483, | |
| "size_gb": 1.2913766270503402, | |
| "path": "common_crawl/2019-30/en_middle_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2223202700, | |
| "size_mb": 2120.2113151550293, | |
| "size_gb": 2.0705188624560833, | |
| "path": "common_crawl/2023-06/en_head_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2207735510, | |
| "size_mb": 2105.4606533050537, | |
| "size_gb": 2.0561139192432165, | |
| "path": "common_crawl/2023-06/en_head_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2152696534, | |
| "size_mb": 2052.9713954925537, | |
| "size_gb": 2.004854878410697, | |
| "path": "common_crawl/2023-06/en_head_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2141264561, | |
| "size_mb": 2042.069016456604, | |
| "size_gb": 1.9942080238834023, | |
| "path": "common_crawl/2019-30/en_head_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2169753773, | |
| "size_mb": 2069.2384462356567, | |
| "size_gb": 2.0207406701520085, | |
| "path": "common_crawl/2023-06/en_head_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2178893815, | |
| "size_mb": 2077.955069541931, | |
| "size_gb": 2.029252997599542, | |
| "path": "common_crawl/2019-30/en_head_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2104683410, | |
| "size_mb": 2007.1825122833252, | |
| "size_gb": 1.9601391721516848, | |
| "path": "common_crawl/2023-06/en_head_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2184513339, | |
| "size_mb": 2083.3142652511597, | |
| "size_gb": 2.0344865871593356, | |
| "path": "common_crawl/2023-06/en_head_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2136785463, | |
| "size_mb": 2037.7974157333374, | |
| "size_gb": 1.9900365388020873, | |
| "path": "common_crawl/2019-30/en_head_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1404703793, | |
| "size_mb": 1339.6299295425415, | |
| "size_gb": 1.3082323530688882, | |
| "path": "common_crawl/2019-30/en_middle_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2179300505, | |
| "size_mb": 2078.3429193496704, | |
| "size_gb": 2.0296317571774125, | |
| "path": "common_crawl/2019-30/en_head_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2160169943, | |
| "size_mb": 2060.098593711853, | |
| "size_gb": 2.0118150329217315, | |
| "path": "common_crawl/2023-06/en_head_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2195564197, | |
| "size_mb": 2093.853184700012, | |
| "size_gb": 2.0447785006836057, | |
| "path": "common_crawl/2019-30/en_head_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2196216267, | |
| "size_mb": 2094.4750471115112, | |
| "size_gb": 2.045385788194835, | |
| "path": "common_crawl/2023-06/en_head_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2209012510, | |
| "size_mb": 2106.6784954071045, | |
| "size_gb": 2.0573032181710005, | |
| "path": "common_crawl/2023-06/en_head_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2210390476, | |
| "size_mb": 2107.9926261901855, | |
| "size_gb": 2.058586549013853, | |
| "path": "common_crawl/2023-06/en_head_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2159189057, | |
| "size_mb": 2059.1631479263306, | |
| "size_gb": 2.010901511646807, | |
| "path": "common_crawl/2019-30/en_head_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2273452689, | |
| "size_mb": 2168.133439064026, | |
| "size_gb": 2.1173178115859628, | |
| "path": "common_crawl/2023-06/en_head_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2191514148, | |
| "size_mb": 2089.9907569885254, | |
| "size_gb": 2.041006598621607, | |
| "path": "common_crawl/2023-06/en_head_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00210-of-01024.jsonl", | |
| "size": 847367530, | |
| "size_mb": 808.1126499176025, | |
| "size_gb": 0.7891725096851587, | |
| "path": "c4/c4-train.00210-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2128790982, | |
| "size_mb": 2030.1732845306396, | |
| "size_gb": 1.9825910981744528, | |
| "path": "common_crawl/2019-30/en_head_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00211-of-01024.jsonl", | |
| "size": 850133755, | |
| "size_mb": 810.7507276535034, | |
| "size_gb": 0.7917487574741244, | |
| "path": "c4/c4-train.00211-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1371785553, | |
| "size_mb": 1308.2366495132446, | |
| "size_gb": 1.277574853040278, | |
| "path": "common_crawl/2019-30/en_middle_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1454363858, | |
| "size_mb": 1386.9894580841064, | |
| "size_gb": 1.3544818926602602, | |
| "path": "common_crawl/2019-30/en_middle_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1469173155, | |
| "size_mb": 1401.1127042770386, | |
| "size_gb": 1.3682741252705455, | |
| "path": "common_crawl/2022-05/en_middle_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1389011510, | |
| "size_mb": 1324.664602279663, | |
| "size_gb": 1.2936177756637335, | |
| "path": "common_crawl/2019-30/en_middle_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1558786822, | |
| "size_mb": 1486.574956893921, | |
| "size_gb": 1.4517333563417196, | |
| "path": "common_crawl/2021-04/en_middle_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_26a6fa61c5eb4bb885e7bc643e285f0e.sampled.jsonl", | |
| "size": 1703319315, | |
| "size_mb": 1624.411883354187, | |
| "size_gb": 1.5863397298380733, | |
| "path": "github/filtered_26a6fa61c5eb4bb885e7bc643e285f0e.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_216883d3a669406699428bc485a4c228.sampled.jsonl", | |
| "size": 1896289353, | |
| "size_mb": 1808.442452430725, | |
| "size_gb": 1.76605708245188, | |
| "path": "github/filtered_216883d3a669406699428bc485a4c228.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00213-of-01024.jsonl", | |
| "size": 849164729, | |
| "size_mb": 809.8265924453735, | |
| "size_gb": 0.7908462816849351, | |
| "path": "c4/c4-train.00213-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1583413983, | |
| "size_mb": 1510.0612478256226, | |
| "size_gb": 1.4746691873297095, | |
| "path": "common_crawl/2021-04/en_middle_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1423497988, | |
| "size_mb": 1357.5534706115723, | |
| "size_gb": 1.3257358111441135, | |
| "path": "common_crawl/2019-30/en_middle_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00212-of-01024.jsonl", | |
| "size": 846509788, | |
| "size_mb": 807.2946434020996, | |
| "size_gb": 0.7883736751973629, | |
| "path": "c4/c4-train.00212-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_25989b8233a04ac791b0eccd502e0c7a.sampled.jsonl", | |
| "size": 1751450062, | |
| "size_mb": 1670.3129405975342, | |
| "size_gb": 1.6311649810522795, | |
| "path": "github/filtered_25989b8233a04ac791b0eccd502e0c7a.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00214-of-01024.jsonl", | |
| "size": 849020699, | |
| "size_mb": 809.6892347335815, | |
| "size_gb": 0.7907121432945132, | |
| "path": "c4/c4-train.00214-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00357-of-01024.jsonl", | |
| "size": 846274177, | |
| "size_mb": 807.0699472427368, | |
| "size_gb": 0.7881542453542352, | |
| "path": "c4/c4-train.00357-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1430575406, | |
| "size_mb": 1364.3030223846436, | |
| "size_gb": 1.3323271702975035, | |
| "path": "common_crawl/2019-30/en_middle_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1432099536, | |
| "size_mb": 1365.7565460205078, | |
| "size_gb": 1.3337466269731522, | |
| "path": "common_crawl/2019-30/en_middle_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00224-of-01024.jsonl", | |
| "size": 845060785, | |
| "size_mb": 805.912766456604, | |
| "size_gb": 0.7870241859927773, | |
| "path": "c4/c4-train.00224-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0078.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1486831922, | |
| "size_mb": 1417.9534168243408, | |
| "size_gb": 1.3847201336175203, | |
| "path": "common_crawl/2023-06/en_middle_0078.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1349229917, | |
| "size_mb": 1286.7259187698364, | |
| "size_gb": 1.2565682800486684, | |
| "path": "common_crawl/2019-30/en_middle_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1426427551, | |
| "size_mb": 1360.3473196029663, | |
| "size_gb": 1.3284641792997718, | |
| "path": "common_crawl/2019-30/en_middle_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1455403563, | |
| "size_mb": 1387.9809980392456, | |
| "size_gb": 1.3554501933977008, | |
| "path": "common_crawl/2022-05/en_middle_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1387780003, | |
| "size_mb": 1323.4901456832886, | |
| "size_gb": 1.2924708453938365, | |
| "path": "common_crawl/2019-30/en_middle_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2060706273, | |
| "size_mb": 1965.2426462173462, | |
| "size_gb": 1.9191822716966271, | |
| "path": "common_crawl/2022-05/en_head_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1538227342, | |
| "size_mb": 1466.967908859253, | |
| "size_gb": 1.4325858484953642, | |
| "path": "common_crawl/2023-06/en_middle_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2202560722, | |
| "size_mb": 2100.5255908966064, | |
| "size_gb": 2.0512945223599672, | |
| "path": "common_crawl/2023-06/en_head_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1583702602, | |
| "size_mb": 1510.3364963531494, | |
| "size_gb": 1.4749379847198725, | |
| "path": "common_crawl/2021-04/en_middle_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1535484947, | |
| "size_mb": 1464.352557182312, | |
| "size_gb": 1.4300317941233516, | |
| "path": "common_crawl/2022-05/en_middle_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2175051481, | |
| "size_mb": 2074.2907342910767, | |
| "size_gb": 2.0256745452061296, | |
| "path": "common_crawl/2022-05/en_head_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1581174468, | |
| "size_mb": 1507.925479888916, | |
| "size_gb": 1.4725834764540195, | |
| "path": "common_crawl/2021-04/en_middle_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1456849278, | |
| "size_mb": 1389.3597393035889, | |
| "size_gb": 1.356796620413661, | |
| "path": "common_crawl/2022-05/en_middle_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1852497974, | |
| "size_mb": 1766.679738998413, | |
| "size_gb": 1.7252731826156378, | |
| "path": "common_crawl/2021-04/en_head_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1526675826, | |
| "size_mb": 1455.951524734497, | |
| "size_gb": 1.4218276608735323, | |
| "path": "common_crawl/2022-05/en_middle_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1952039674, | |
| "size_mb": 1861.610101699829, | |
| "size_gb": 1.8179786149412394, | |
| "path": "common_crawl/2021-04/en_head_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2194591971, | |
| "size_mb": 2092.92599773407, | |
| "size_gb": 2.0438730446621776, | |
| "path": "common_crawl/2022-05/en_head_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1905338395, | |
| "size_mb": 1817.0722913742065, | |
| "size_gb": 1.7744846595451236, | |
| "path": "common_crawl/2021-04/en_head_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1843288301, | |
| "size_mb": 1757.896710395813, | |
| "size_gb": 1.7166960062459111, | |
| "path": "common_crawl/2021-04/en_head_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_34b0785b77814b7583433ddb27a61ae0.sampled.jsonl", | |
| "size": 3381920273, | |
| "size_mb": 3225.2505044937134, | |
| "size_gb": 3.149658695794642, | |
| "path": "github/filtered_34b0785b77814b7583433ddb27a61ae0.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1419087700, | |
| "size_mb": 1353.3474922180176, | |
| "size_gb": 1.3216284103691578, | |
| "path": "common_crawl/2019-30/en_middle_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00223-of-01024.jsonl", | |
| "size": 846255630, | |
| "size_mb": 807.0522594451904, | |
| "size_gb": 0.7881369721144438, | |
| "path": "c4/c4-train.00223-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_28106e66bfd94978abbc15ec845aeddb.sampled.jsonl", | |
| "size": 2989082439, | |
| "size_mb": 2850.6111516952515, | |
| "size_gb": 2.783799952827394, | |
| "path": "github/filtered_28106e66bfd94978abbc15ec845aeddb.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00218-of-01024.jsonl", | |
| "size": 845259502, | |
| "size_mb": 806.1022777557373, | |
| "size_gb": 0.7872092556208372, | |
| "path": "c4/c4-train.00218-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00219-of-01024.jsonl", | |
| "size": 846258830, | |
| "size_mb": 807.0553112030029, | |
| "size_gb": 0.7881399523466825, | |
| "path": "c4/c4-train.00219-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_3be6a5cc7428401393c23e5516a43537.sampled.jsonl", | |
| "size": 3117013413, | |
| "size_mb": 2972.615635871887, | |
| "size_gb": 2.90294495690614, | |
| "path": "github/filtered_3be6a5cc7428401393c23e5516a43537.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_275100c1a44f4451b0343373ebc5637a.sampled.jsonl", | |
| "size": 4670547664, | |
| "size_mb": 4454.181350708008, | |
| "size_gb": 4.349786475300789, | |
| "path": "github/filtered_275100c1a44f4451b0343373ebc5637a.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00221-of-01024.jsonl", | |
| "size": 844525405, | |
| "size_mb": 805.4021883010864, | |
| "size_gb": 0.7865255745127797, | |
| "path": "c4/c4-train.00221-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_30d01c4edab64866bda8c609e90b4f4e.sampled.jsonl", | |
| "size": 3502260170, | |
| "size_mb": 3340.015573501587, | |
| "size_gb": 3.2617339584976435, | |
| "path": "github/filtered_30d01c4edab64866bda8c609e90b4f4e.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_24278a707deb445b8e4f59c83dd67910.sampled.jsonl", | |
| "size": 1616017898, | |
| "size_mb": 1541.154764175415, | |
| "size_gb": 1.5050339493900537, | |
| "path": "github/filtered_24278a707deb445b8e4f59c83dd67910.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00220-of-01024.jsonl", | |
| "size": 842184106, | |
| "size_mb": 803.1693515777588, | |
| "size_gb": 0.7843450699001551, | |
| "path": "c4/c4-train.00220-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00222-of-01024.jsonl", | |
| "size": 845678227, | |
| "size_mb": 806.5016050338745, | |
| "size_gb": 0.7875992236658931, | |
| "path": "c4/c4-train.00222-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_2afd093fefad4c8da76cc539e8fb6137.sampled.jsonl", | |
| "size": 2151014531, | |
| "size_mb": 2051.3673124313354, | |
| "size_gb": 2.003288391046226, | |
| "path": "github/filtered_2afd093fefad4c8da76cc539e8fb6137.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00215-of-01024.jsonl", | |
| "size": 850094486, | |
| "size_mb": 810.7132778167725, | |
| "size_gb": 0.7917121853679419, | |
| "path": "c4/c4-train.00215-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00216-of-01024.jsonl", | |
| "size": 848568087, | |
| "size_mb": 809.2575902938843, | |
| "size_gb": 0.7902906155213714, | |
| "path": "c4/c4-train.00216-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00217-of-01024.jsonl", | |
| "size": 849955113, | |
| "size_mb": 810.580361366272, | |
| "size_gb": 0.79158238414675, | |
| "path": "c4/c4-train.00217-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_366012588bef4d749bbbea76ae701141.sampled.jsonl", | |
| "size": 1565332841, | |
| "size_mb": 1492.8177270889282, | |
| "size_gb": 1.4578298116102815, | |
| "path": "github/filtered_366012588bef4d749bbbea76ae701141.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_27f05c041a1c401783f90b9415e40e4b.sampled.jsonl", | |
| "size": 2022347763, | |
| "size_mb": 1928.661120414734, | |
| "size_gb": 1.8834581254050136, | |
| "path": "github/filtered_27f05c041a1c401783f90b9415e40e4b.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_34e78793c1e94eeebd92852399097596.sampled.jsonl", | |
| "size": 2190228716, | |
| "size_mb": 2088.7648735046387, | |
| "size_gb": 2.0398094467818737, | |
| "path": "github/filtered_34e78793c1e94eeebd92852399097596.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2148760756, | |
| "size_mb": 2049.217945098877, | |
| "size_gb": 2.001189399510622, | |
| "path": "common_crawl/2022-05/en_head_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_36c2a955ddd84672bbc778aa4ad2fbaf.sampled.jsonl", | |
| "size": 2439873921, | |
| "size_mb": 2326.845093727112, | |
| "size_gb": 2.2723096618428826, | |
| "path": "github/filtered_36c2a955ddd84672bbc778aa4ad2fbaf.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1612878339, | |
| "size_mb": 1538.160647392273, | |
| "size_gb": 1.5021100072190166, | |
| "path": "common_crawl/2021-04/en_middle_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1629114693, | |
| "size_mb": 1553.6448411941528, | |
| "size_gb": 1.5172312902286649, | |
| "path": "common_crawl/2021-04/en_middle_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1561353775, | |
| "size_mb": 1489.0229940414429, | |
| "size_gb": 1.4541240176185966, | |
| "path": "common_crawl/2021-04/en_middle_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1874421446, | |
| "size_mb": 1787.5875911712646, | |
| "size_gb": 1.7456910070031881, | |
| "path": "common_crawl/2021-04/en_head_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1500662141, | |
| "size_mb": 1431.1429414749146, | |
| "size_gb": 1.3976005287840962, | |
| "path": "common_crawl/2022-05/en_middle_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1481519749, | |
| "size_mb": 1412.887333869934, | |
| "size_gb": 1.3797727869823575, | |
| "path": "common_crawl/2022-05/en_middle_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1579430566, | |
| "size_mb": 1506.2623653411865, | |
| "size_gb": 1.4709593411535025, | |
| "path": "common_crawl/2021-04/en_middle_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2193523039, | |
| "size_mb": 2091.906584739685, | |
| "size_gb": 2.0428775241598487, | |
| "path": "common_crawl/2022-05/en_head_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1395188234, | |
| "size_mb": 1330.5551853179932, | |
| "size_gb": 1.2993702981621027, | |
| "path": "common_crawl/2019-30/en_middle_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1578422798, | |
| "size_mb": 1505.3012828826904, | |
| "size_gb": 1.4700207840651274, | |
| "path": "common_crawl/2021-04/en_middle_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1584679257, | |
| "size_mb": 1511.2679071426392, | |
| "size_gb": 1.4758475655689836, | |
| "path": "common_crawl/2021-04/en_middle_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1837583012, | |
| "size_mb": 1752.455722808838, | |
| "size_gb": 1.7113825418055058, | |
| "path": "common_crawl/2021-04/en_head_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1813256109, | |
| "size_mb": 1729.2557802200317, | |
| "size_gb": 1.6887263478711247, | |
| "path": "common_crawl/2021-04/en_head_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1844306770, | |
| "size_mb": 1758.867998123169, | |
| "size_gb": 1.7176445294171572, | |
| "path": "common_crawl/2021-04/en_head_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1905655640, | |
| "size_mb": 1817.3748397827148, | |
| "size_gb": 1.7747801169753075, | |
| "path": "common_crawl/2021-04/en_head_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00358-of-01024.jsonl", | |
| "size": 843853393, | |
| "size_mb": 804.7613077163696, | |
| "size_gb": 0.7858997145667672, | |
| "path": "c4/c4-train.00358-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_483f11d6bc864f7fbfbe63bdf3583ce2.sampled.jsonl", | |
| "size": 1998410290, | |
| "size_mb": 1905.8325672149658, | |
| "size_gb": 1.861164616420865, | |
| "path": "github/filtered_483f11d6bc864f7fbfbe63bdf3583ce2.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_4d44caf43c154ae4aaeab36eab0221c9.sampled.jsonl", | |
| "size": 1667192326, | |
| "size_mb": 1589.958501815796, | |
| "size_gb": 1.5526938494294882, | |
| "path": "github/filtered_4d44caf43c154ae4aaeab36eab0221c9.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1627767460, | |
| "size_mb": 1552.360019683838, | |
| "size_gb": 1.515976581722498, | |
| "path": "common_crawl/2021-04/en_middle_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_4bebabdbd8544da7a2071864ccf81f2e.sampled.jsonl", | |
| "size": 1654984424, | |
| "size_mb": 1578.3161392211914, | |
| "size_gb": 1.5413243547081947, | |
| "path": "github/filtered_4bebabdbd8544da7a2071864ccf81f2e.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_b6c2e4d3-d215-4d99-891f-d20b997d4d5a.jsonl", | |
| "size": 906853980, | |
| "size_mb": 864.8433494567871, | |
| "size_gb": 0.8445735834538937, | |
| "path": "arxiv/arxiv_b6c2e4d3-d215-4d99-891f-d20b997d4d5a.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_4b6883dc304c4e799620ec95b96dc91a.sampled.jsonl", | |
| "size": 1726735384, | |
| "size_mb": 1646.7431869506836, | |
| "size_gb": 1.608147643506527, | |
| "path": "github/filtered_4b6883dc304c4e799620ec95b96dc91a.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1506775560, | |
| "size_mb": 1436.9731521606445, | |
| "size_gb": 1.4032940939068794, | |
| "path": "common_crawl/2022-05/en_middle_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_4f98dc136ba94eeaa1ba6c974814b33c.sampled.jsonl", | |
| "size": 2270309835, | |
| "size_mb": 2165.1361799240112, | |
| "size_gb": 2.114390800707042, | |
| "path": "github/filtered_4f98dc136ba94eeaa1ba6c974814b33c.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1520354103, | |
| "size_mb": 1449.9226598739624, | |
| "size_gb": 1.4159400975331664, | |
| "path": "common_crawl/2022-05/en_middle_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_4ff3604761614a3db550ef758e6457b5.sampled.jsonl", | |
| "size": 2191628537, | |
| "size_mb": 2090.099846839905, | |
| "size_gb": 2.0411131316795945, | |
| "path": "github/filtered_4ff3604761614a3db550ef758e6457b5.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00009-of-01024.jsonl", | |
| "size": 843054511, | |
| "size_mb": 803.9994344711304, | |
| "size_gb": 0.7851556977257133, | |
| "path": "c4/c4-train.00009-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1588836953, | |
| "size_mb": 1515.2329950332642, | |
| "size_gb": 1.479719721712172, | |
| "path": "common_crawl/2021-04/en_middle_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1485960771, | |
| "size_mb": 1417.1226224899292, | |
| "size_gb": 1.3839088110253215, | |
| "path": "common_crawl/2022-05/en_middle_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1904177101, | |
| "size_mb": 1815.9647951126099, | |
| "size_gb": 1.773403120227158, | |
| "path": "common_crawl/2021-04/en_head_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1487606247, | |
| "size_mb": 1418.691870689392, | |
| "size_gb": 1.3854412799701095, | |
| "path": "common_crawl/2022-05/en_middle_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2167792214, | |
| "size_mb": 2067.367757797241, | |
| "size_gb": 2.0189138259738684, | |
| "path": "common_crawl/2023-06/en_head_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1453964442, | |
| "size_mb": 1386.6085453033447, | |
| "size_gb": 1.3541099075227976, | |
| "path": "common_crawl/2022-05/en_middle_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1854319800, | |
| "size_mb": 1768.4171676635742, | |
| "size_gb": 1.7269698902964592, | |
| "path": "common_crawl/2021-04/en_head_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00228-of-01024.jsonl", | |
| "size": 845273064, | |
| "size_mb": 806.1152114868164, | |
| "size_gb": 0.7872218862175941, | |
| "path": "c4/c4-train.00228-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1505772824, | |
| "size_mb": 1436.0168685913086, | |
| "size_gb": 1.4023602232336998, | |
| "path": "common_crawl/2022-05/en_middle_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1840650889, | |
| "size_mb": 1755.3814783096313, | |
| "size_gb": 1.7142397249117494, | |
| "path": "common_crawl/2021-04/en_head_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1872597341, | |
| "size_mb": 1785.8479890823364, | |
| "size_gb": 1.7439921768382192, | |
| "path": "common_crawl/2021-04/en_head_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00226-of-01024.jsonl", | |
| "size": 842874736, | |
| "size_mb": 803.8279876708984, | |
| "size_gb": 0.7849882692098618, | |
| "path": "c4/c4-train.00226-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_3cc6cab9266746a6befa23648aa43119.sampled.jsonl", | |
| "size": 1750936058, | |
| "size_mb": 1669.822748184204, | |
| "size_gb": 1.6306862775236368, | |
| "path": "github/filtered_3cc6cab9266746a6befa23648aa43119.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00235-of-01024.jsonl", | |
| "size": 841584298, | |
| "size_mb": 802.5973300933838, | |
| "size_gb": 0.7837864551693201, | |
| "path": "c4/c4-train.00235-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00236-of-01024.jsonl", | |
| "size": 844230853, | |
| "size_mb": 805.1212816238403, | |
| "size_gb": 0.7862512515857816, | |
| "path": "c4/c4-train.00236-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1594248956, | |
| "size_mb": 1520.3942832946777, | |
| "size_gb": 1.4847600422799587, | |
| "path": "common_crawl/2021-04/en_middle_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_58fad5994b4446c6bceb33453484acb4.sampled.jsonl", | |
| "size": 2579005549, | |
| "size_mb": 2459.5313539505005, | |
| "size_gb": 2.4018860878422856, | |
| "path": "github/filtered_58fad5994b4446c6bceb33453484acb4.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_50423e84046948b4a2a70e7e4538e12d.sampled.jsonl", | |
| "size": 1701335896, | |
| "size_mb": 1622.5203475952148, | |
| "size_gb": 1.584492526948452, | |
| "path": "github/filtered_50423e84046948b4a2a70e7e4538e12d.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_21078cf63afb4d9eb4a7876f726a7226.sampled.jsonl", | |
| "size": 1674939745, | |
| "size_mb": 1597.3470163345337, | |
| "size_gb": 1.559909195639193, | |
| "path": "github/filtered_21078cf63afb4d9eb4a7876f726a7226.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_478afe8aaccb43e6be2de7e34e041ef3.sampled.jsonl", | |
| "size": 2049446485, | |
| "size_mb": 1954.504475593567, | |
| "size_gb": 1.9086957769468427, | |
| "path": "github/filtered_478afe8aaccb43e6be2de7e34e041ef3.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_3f09f384f0734a4b912bb75db3f812bc.sampled.jsonl", | |
| "size": 2620005409, | |
| "size_mb": 2498.6318674087524, | |
| "size_gb": 2.44007018301636, | |
| "path": "github/filtered_3f09f384f0734a4b912bb75db3f812bc.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1870681315, | |
| "size_mb": 1784.0207242965698, | |
| "size_gb": 1.742207738570869, | |
| "path": "common_crawl/2021-04/en_head_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2195565337, | |
| "size_mb": 2093.854271888733, | |
| "size_gb": 2.0447795623913407, | |
| "path": "common_crawl/2022-05/en_head_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2187003433, | |
| "size_mb": 2085.689003944397, | |
| "size_gb": 2.03680566791445, | |
| "path": "common_crawl/2022-05/en_head_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00227-of-01024.jsonl", | |
| "size": 849166538, | |
| "size_mb": 809.8283176422119, | |
| "size_gb": 0.7908479664474726, | |
| "path": "c4/c4-train.00227-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1368842432, | |
| "size_mb": 1305.4298706054688, | |
| "size_gb": 1.274833858013153, | |
| "path": "common_crawl/2022-05/en_middle_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1505603199, | |
| "size_mb": 1435.8551015853882, | |
| "size_gb": 1.4022022476419806, | |
| "path": "common_crawl/2022-05/en_middle_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2128149650, | |
| "size_mb": 2029.5616626739502, | |
| "size_gb": 1.9819938112050295, | |
| "path": "common_crawl/2022-05/en_head_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2103733118, | |
| "size_mb": 2006.2762432098389, | |
| "size_gb": 1.9592541437596083, | |
| "path": "common_crawl/2022-05/en_head_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2256041325, | |
| "size_mb": 2151.5286684036255, | |
| "size_gb": 2.1011022152379155, | |
| "path": "common_crawl/2022-05/en_head_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00225-of-01024.jsonl", | |
| "size": 846824333, | |
| "size_mb": 807.5946168899536, | |
| "size_gb": 0.7886666180565953, | |
| "path": "c4/c4-train.00225-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1796112181, | |
| "size_mb": 1712.9060564041138, | |
| "size_gb": 1.6727598207071424, | |
| "path": "common_crawl/2021-04/en_head_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2027666869, | |
| "size_mb": 1933.7338151931763, | |
| "size_gb": 1.8884119288995862, | |
| "path": "common_crawl/2022-05/en_head_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1808805530, | |
| "size_mb": 1725.0113773345947, | |
| "size_gb": 1.6845814231783152, | |
| "path": "common_crawl/2021-04/en_head_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00241-of-01024.jsonl", | |
| "size": 847177523, | |
| "size_mb": 807.9314451217651, | |
| "size_gb": 0.7889955518767238, | |
| "path": "c4/c4-train.00241-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1839654006, | |
| "size_mb": 1754.4307765960693, | |
| "size_gb": 1.713311305269599, | |
| "path": "common_crawl/2021-04/en_head_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1556172247, | |
| "size_mb": 1484.081503868103, | |
| "size_gb": 1.4492983436211944, | |
| "path": "common_crawl/2022-05/en_middle_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1852628227, | |
| "size_mb": 1766.803957939148, | |
| "size_gb": 1.7253944901749492, | |
| "path": "common_crawl/2021-04/en_head_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2212762519, | |
| "size_mb": 2110.254782676697, | |
| "size_gb": 2.0607956862077117, | |
| "path": "common_crawl/2022-05/en_head_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2157815347, | |
| "size_mb": 2057.85307598114, | |
| "size_gb": 2.009622144512832, | |
| "path": "common_crawl/2023-06/en_head_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1856618879, | |
| "size_mb": 1770.6097402572632, | |
| "size_gb": 1.7291110744699836, | |
| "path": "common_crawl/2021-04/en_head_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1403104974, | |
| "size_mb": 1338.1051769256592, | |
| "size_gb": 1.306743336841464, | |
| "path": "common_crawl/2022-05/en_middle_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_6742e99802894114a3ba44841f49b168.sampled.jsonl", | |
| "size": 1546889066, | |
| "size_mb": 1475.2283725738525, | |
| "size_gb": 1.4406527075916529, | |
| "path": "github/filtered_6742e99802894114a3ba44841f49b168.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1523529970, | |
| "size_mb": 1452.9514026641846, | |
| "size_gb": 1.4188978541642427, | |
| "path": "common_crawl/2022-05/en_middle_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2209388722, | |
| "size_mb": 2107.0372791290283, | |
| "size_gb": 2.0576535928994417, | |
| "path": "common_crawl/2022-05/en_head_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1332577315, | |
| "size_mb": 1270.844759941101, | |
| "size_gb": 1.2410593358799815, | |
| "path": "common_crawl/2022-05/en_middle_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00257-of-01024.jsonl", | |
| "size": 842681297, | |
| "size_mb": 803.6435098648071, | |
| "size_gb": 0.7848081151023507, | |
| "path": "c4/c4-train.00257-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1872606668, | |
| "size_mb": 1785.8568840026855, | |
| "size_gb": 1.7440008632838726, | |
| "path": "common_crawl/2021-04/en_head_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2106769926, | |
| "size_mb": 2009.172369003296, | |
| "size_gb": 1.9620823916047812, | |
| "path": "common_crawl/2022-05/en_head_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1852782916, | |
| "size_mb": 1766.9514808654785, | |
| "size_gb": 1.7255385555326939, | |
| "path": "common_crawl/2021-04/en_head_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1588461390, | |
| "size_mb": 1514.8748302459717, | |
| "size_gb": 1.4793699514120817, | |
| "path": "common_crawl/2021-04/en_middle_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1559031910, | |
| "size_mb": 1486.8086910247803, | |
| "size_gb": 1.451961612328887, | |
| "path": "common_crawl/2021-04/en_middle_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2152003278, | |
| "size_mb": 2052.310255050659, | |
| "size_gb": 2.0042092334479094, | |
| "path": "common_crawl/2023-06/en_head_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2213858269, | |
| "size_mb": 2111.299771308899, | |
| "size_gb": 2.0618161829188466, | |
| "path": "common_crawl/2023-06/en_head_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2138546351, | |
| "size_mb": 2039.4767293930054, | |
| "size_gb": 1.9916764935478568, | |
| "path": "common_crawl/2023-06/en_head_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1447571269, | |
| "size_mb": 1380.5115404129028, | |
| "size_gb": 1.3481558011844754, | |
| "path": "common_crawl/2022-05/en_middle_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1603952969, | |
| "size_mb": 1529.64875125885, | |
| "size_gb": 1.4937976086512208, | |
| "path": "common_crawl/2021-04/en_middle_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01018-of-01024.jsonl", | |
| "size": 847349500, | |
| "size_mb": 808.0954551696777, | |
| "size_gb": 0.7891557179391384, | |
| "path": "c4/c4-train.01018-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1814594884, | |
| "size_mb": 1730.5325355529785, | |
| "size_gb": 1.6899731792509556, | |
| "path": "common_crawl/2021-04/en_head_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1864131611, | |
| "size_mb": 1777.7744398117065, | |
| "size_gb": 1.7361078513786197, | |
| "path": "common_crawl/2021-04/en_head_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1558281049, | |
| "size_mb": 1486.0926141738892, | |
| "size_gb": 1.4512623185291886, | |
| "path": "common_crawl/2021-04/en_middle_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1864744527, | |
| "size_mb": 1778.358962059021, | |
| "size_gb": 1.7366786738857627, | |
| "path": "common_crawl/2021-04/en_head_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2163116215, | |
| "size_mb": 2062.9083776474, | |
| "size_gb": 2.014558962546289, | |
| "path": "common_crawl/2022-05/en_head_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00813-of-01024.jsonl", | |
| "size": 845559437, | |
| "size_mb": 806.3883180618286, | |
| "size_gb": 0.7874885918572545, | |
| "path": "c4/c4-train.00813-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1477644881, | |
| "size_mb": 1409.1919717788696, | |
| "size_gb": 1.3761640349403024, | |
| "path": "common_crawl/2022-05/en_middle_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1801383589, | |
| "size_mb": 1717.9332628250122, | |
| "size_gb": 1.677669201977551, | |
| "path": "common_crawl/2021-04/en_head_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2082757364, | |
| "size_mb": 1986.2722053527832, | |
| "size_gb": 1.9397189505398273, | |
| "path": "common_crawl/2022-05/en_head_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00293-of-01024.jsonl", | |
| "size": 848395292, | |
| "size_mb": 809.0928001403809, | |
| "size_gb": 0.7901296876370907, | |
| "path": "c4/c4-train.00293-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00333-of-01024.jsonl", | |
| "size": 849533811, | |
| "size_mb": 810.1785764694214, | |
| "size_gb": 0.7911900160834193, | |
| "path": "c4/c4-train.00333-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00772-of-01024.jsonl", | |
| "size": 847488310, | |
| "size_mb": 808.2278347015381, | |
| "size_gb": 0.7892849948257208, | |
| "path": "c4/c4-train.00772-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1416834045, | |
| "size_mb": 1351.198239326477, | |
| "size_gb": 1.3195295305922627, | |
| "path": "common_crawl/2022-05/en_middle_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2268336707, | |
| "size_mb": 2163.254458427429, | |
| "size_gb": 2.1125531820580363, | |
| "path": "common_crawl/2023-06/en_head_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_eabf427d56184fb89f9b5f27e73f7988.sampled.jsonl", | |
| "size": 1960567466, | |
| "size_mb": 1869.7428379058838, | |
| "size_gb": 1.8259207401424646, | |
| "path": "github/filtered_eabf427d56184fb89f9b5f27e73f7988.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1629991698, | |
| "size_mb": 1554.4812183380127, | |
| "size_gb": 1.5180480647832155, | |
| "path": "common_crawl/2021-04/en_middle_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2173295681, | |
| "size_mb": 2072.6162729263306, | |
| "size_gb": 2.0240393290296197, | |
| "path": "common_crawl/2023-06/en_head_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_2c131fca-2a05-4d5f-a805-59d2af3477e2.jsonl", | |
| "size": 1001832019, | |
| "size_mb": 955.4214658737183, | |
| "size_gb": 0.933028775267303, | |
| "path": "arxiv/arxiv_2c131fca-2a05-4d5f-a805-59d2af3477e2.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00562-of-01024.jsonl", | |
| "size": 846706856, | |
| "size_mb": 807.4825820922852, | |
| "size_gb": 0.7885572090744972, | |
| "path": "c4/c4-train.00562-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1863361168, | |
| "size_mb": 1777.0396881103516, | |
| "size_gb": 1.7353903204202652, | |
| "path": "common_crawl/2021-04/en_head_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1935992573, | |
| "size_mb": 1846.306393623352, | |
| "size_gb": 1.8030335875228047, | |
| "path": "common_crawl/2021-04/en_head_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1417965550, | |
| "size_mb": 1352.2773265838623, | |
| "size_gb": 1.320583326742053, | |
| "path": "common_crawl/2022-05/en_middle_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1585163765, | |
| "size_mb": 1511.7299699783325, | |
| "size_gb": 1.4762987988069654, | |
| "path": "common_crawl/2021-04/en_middle_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1459817653, | |
| "size_mb": 1392.1906023025513, | |
| "size_gb": 1.3595611350610852, | |
| "path": "common_crawl/2022-05/en_middle_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1429579349, | |
| "size_mb": 1363.353108406067, | |
| "size_gb": 1.3313995199277997, | |
| "path": "common_crawl/2022-05/en_middle_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1567314053, | |
| "size_mb": 1494.707158088684, | |
| "size_gb": 1.4596749590709805, | |
| "path": "common_crawl/2022-05/en_middle_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00775-of-01024.jsonl", | |
| "size": 849735773, | |
| "size_mb": 810.3711824417114, | |
| "size_gb": 0.7913781078532338, | |
| "path": "c4/c4-train.00775-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1546649199, | |
| "size_mb": 1474.9996175765991, | |
| "size_gb": 1.4404293140396476, | |
| "path": "common_crawl/2021-04/en_middle_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2064362370, | |
| "size_mb": 1968.7293720245361, | |
| "size_gb": 1.922587277367711, | |
| "path": "common_crawl/2023-06/en_head_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1523125937, | |
| "size_mb": 1452.566086769104, | |
| "size_gb": 1.4185215691104531, | |
| "path": "common_crawl/2022-05/en_middle_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1304222293, | |
| "size_mb": 1243.803303718567, | |
| "size_gb": 1.214651663787663, | |
| "path": "common_crawl/2020-05/en_middle_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1440552398, | |
| "size_mb": 1373.8178234100342, | |
| "size_gb": 1.3416189681738615, | |
| "path": "common_crawl/2022-05/en_middle_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1862744487, | |
| "size_mb": 1776.4515752792358, | |
| "size_gb": 1.7348159914836287, | |
| "path": "common_crawl/2021-04/en_head_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2213265746, | |
| "size_mb": 2110.734697341919, | |
| "size_gb": 2.0612643528729677, | |
| "path": "common_crawl/2023-06/en_head_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2202206389, | |
| "size_mb": 2100.1876726150513, | |
| "size_gb": 2.050964524038136, | |
| "path": "common_crawl/2023-06/en_head_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1535180352, | |
| "size_mb": 1464.0620727539062, | |
| "size_gb": 1.4297481179237366, | |
| "path": "common_crawl/2022-05/en_middle_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1544189121, | |
| "size_mb": 1472.653504371643, | |
| "size_gb": 1.4381381878629327, | |
| "path": "common_crawl/2021-04/en_middle_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1493431200, | |
| "size_mb": 1424.2469787597656, | |
| "size_gb": 1.3908661901950836, | |
| "path": "common_crawl/2022-05/en_middle_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1391183541, | |
| "size_mb": 1326.7360124588013, | |
| "size_gb": 1.2956406371667981, | |
| "path": "common_crawl/2019-30/en_middle_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00779-of-01024.jsonl", | |
| "size": 845776800, | |
| "size_mb": 806.5956115722656, | |
| "size_gb": 0.7876910269260406, | |
| "path": "c4/c4-train.00779-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01013-of-01024.jsonl", | |
| "size": 844816884, | |
| "size_mb": 805.6801643371582, | |
| "size_gb": 0.7867970354855061, | |
| "path": "c4/c4-train.01013-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00771-of-01024.jsonl", | |
| "size": 845528059, | |
| "size_mb": 806.3583936691284, | |
| "size_gb": 0.7874593688175082, | |
| "path": "c4/c4-train.00771-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_db611a692a704f6db3c18e77f79fd2f0.sampled.jsonl", | |
| "size": 2807297396, | |
| "size_mb": 2677.2474250793457, | |
| "size_gb": 2.6144994385540485, | |
| "path": "github/filtered_db611a692a704f6db3c18e77f79fd2f0.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01012-of-01024.jsonl", | |
| "size": 847142347, | |
| "size_mb": 807.8978986740112, | |
| "size_gb": 0.7889627916738391, | |
| "path": "c4/c4-train.01012-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_73241940-66c1-481c-b53a-f5e8b9afe9fa.jsonl", | |
| "size": 992458250, | |
| "size_mb": 946.4819431304932, | |
| "size_gb": 0.9242987725883722, | |
| "path": "arxiv/arxiv_73241940-66c1-481c-b53a-f5e8b9afe9fa.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00291-of-01024.jsonl", | |
| "size": 845108235, | |
| "size_mb": 805.9580183029175, | |
| "size_gb": 0.7870683772489429, | |
| "path": "c4/c4-train.00291-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1463729712, | |
| "size_mb": 1395.9214324951172, | |
| "size_gb": 1.3632045239210129, | |
| "path": "common_crawl/2022-05/en_middle_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_dcaa5c8b729b4fb599399dbf4557e43e.sampled.jsonl", | |
| "size": 1902033230, | |
| "size_mb": 1813.9202404022217, | |
| "size_gb": 1.7714064847677946, | |
| "path": "github/filtered_dcaa5c8b729b4fb599399dbf4557e43e.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00292-of-01024.jsonl", | |
| "size": 848482368, | |
| "size_mb": 809.1758422851562, | |
| "size_gb": 0.7902107834815979, | |
| "path": "c4/c4-train.00292-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00773-of-01024.jsonl", | |
| "size": 849626135, | |
| "size_mb": 810.2666234970093, | |
| "size_gb": 0.7912759995087981, | |
| "path": "c4/c4-train.00773-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1345051788, | |
| "size_mb": 1282.7413444519043, | |
| "size_gb": 1.2526770941913128, | |
| "path": "common_crawl/2020-05/en_middle_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1295904708, | |
| "size_mb": 1235.871036529541, | |
| "size_gb": 1.20690530911088, | |
| "path": "common_crawl/2020-05/en_middle_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1420191465, | |
| "size_mb": 1354.4001245498657, | |
| "size_gb": 1.3226563716307282, | |
| "path": "common_crawl/2019-30/en_middle_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00774-of-01024.jsonl", | |
| "size": 849280293, | |
| "size_mb": 809.9368028640747, | |
| "size_gb": 0.790953909046948, | |
| "path": "c4/c4-train.00774-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_f931feb0e85940879d194c0e20d9e28a.sampled.jsonl", | |
| "size": 2157247981, | |
| "size_mb": 2057.311993598938, | |
| "size_gb": 2.009093743748963, | |
| "path": "github/filtered_f931feb0e85940879d194c0e20d9e28a.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00297-of-01024.jsonl", | |
| "size": 848744701, | |
| "size_mb": 809.426022529602, | |
| "size_gb": 0.7904551001265645, | |
| "path": "c4/c4-train.00297-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_eee6980922ca4a14b0c3341fa8a904d9.sampled.jsonl", | |
| "size": 3197661312, | |
| "size_mb": 3049.5274658203125, | |
| "size_gb": 2.978054165840149, | |
| "path": "github/filtered_eee6980922ca4a14b0c3341fa8a904d9.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_1caed86f-5625-4941-bdc1-cc57e4fec1cd.jsonl", | |
| "size": 817361027, | |
| "size_mb": 779.4962186813354, | |
| "size_gb": 0.7612267760559916, | |
| "path": "arxiv/arxiv_1caed86f-5625-4941-bdc1-cc57e4fec1cd.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00296-of-01024.jsonl", | |
| "size": 847005703, | |
| "size_mb": 807.7675848007202, | |
| "size_gb": 0.7888355320319533, | |
| "path": "c4/c4-train.00296-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00229-of-01024.jsonl", | |
| "size": 846202829, | |
| "size_mb": 807.0019044876099, | |
| "size_gb": 0.7880877973511815, | |
| "path": "c4/c4-train.00229-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1420397406, | |
| "size_mb": 1354.5965251922607, | |
| "size_gb": 1.3228481691330671, | |
| "path": "common_crawl/2019-30/en_middle_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_85b4c166-469d-449c-ab3d-5214c1d80246.jsonl", | |
| "size": 1000006000, | |
| "size_mb": 953.6800384521484, | |
| "size_gb": 0.9313281625509262, | |
| "path": "arxiv/arxiv_85b4c166-469d-449c-ab3d-5214c1d80246.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_5a02d9ee-12a0-437d-808f-d26f0eb2012b.jsonl", | |
| "size": 1072661263, | |
| "size_mb": 1022.9694967269897, | |
| "size_gb": 0.9989936491474509, | |
| "path": "arxiv/arxiv_5a02d9ee-12a0-437d-808f-d26f0eb2012b.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00776-of-01024.jsonl", | |
| "size": 845013778, | |
| "size_mb": 805.8679370880127, | |
| "size_gb": 0.7869804073125124, | |
| "path": "c4/c4-train.00776-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00778-of-01024.jsonl", | |
| "size": 843162633, | |
| "size_mb": 804.1025476455688, | |
| "size_gb": 0.7852563941851258, | |
| "path": "c4/c4-train.00778-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01017-of-01024.jsonl", | |
| "size": 847084112, | |
| "size_mb": 807.8423614501953, | |
| "size_gb": 0.7889085561037064, | |
| "path": "c4/c4-train.01017-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01016-of-01024.jsonl", | |
| "size": 844093250, | |
| "size_mb": 804.9900531768799, | |
| "size_gb": 0.7861230988055468, | |
| "path": "c4/c4-train.01016-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00298-of-01024.jsonl", | |
| "size": 851090490, | |
| "size_mb": 811.6631412506104, | |
| "size_gb": 0.7926397863775492, | |
| "path": "c4/c4-train.00298-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_50757a42-079b-41ec-bcca-73759faffd62.jsonl", | |
| "size": 1164763960, | |
| "size_mb": 1110.8054733276367, | |
| "size_gb": 1.0847709700465202, | |
| "path": "arxiv/arxiv_50757a42-079b-41ec-bcca-73759faffd62.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01014-of-01024.jsonl", | |
| "size": 842907734, | |
| "size_mb": 803.8594570159912, | |
| "size_gb": 0.7850190009921789, | |
| "path": "c4/c4-train.01014-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00777-of-01024.jsonl", | |
| "size": 847125748, | |
| "size_mb": 807.8820686340332, | |
| "size_gb": 0.788947332650423, | |
| "path": "c4/c4-train.00777-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_f9bbe6a065004a4c8e018c6ad63063b2.sampled.jsonl", | |
| "size": 3278640054, | |
| "size_mb": 3126.7548122406006, | |
| "size_gb": 3.0534714963287115, | |
| "path": "github/filtered_f9bbe6a065004a4c8e018c6ad63063b2.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00465-of-01024.jsonl", | |
| "size": 849203055, | |
| "size_mb": 809.8631429672241, | |
| "size_gb": 0.7908819755539298, | |
| "path": "c4/c4-train.00465-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00470-of-01024.jsonl", | |
| "size": 846307618, | |
| "size_mb": 807.1018390655518, | |
| "size_gb": 0.7881853897124529, | |
| "path": "c4/c4-train.00470-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_f478ff72b57f4f4283c22ac22ae84134.sampled.jsonl", | |
| "size": 1619590378, | |
| "size_mb": 1544.56174659729, | |
| "size_gb": 1.508361080661416, | |
| "path": "github/filtered_f478ff72b57f4f4283c22ac22ae84134.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1309874778, | |
| "size_mb": 1249.1939334869385, | |
| "size_gb": 1.2199159506708384, | |
| "path": "common_crawl/2020-05/en_middle_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00469-of-01024.jsonl", | |
| "size": 844113714, | |
| "size_mb": 805.0095691680908, | |
| "size_gb": 0.7861421573907137, | |
| "path": "c4/c4-train.00469-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1479584714, | |
| "size_mb": 1411.041940689087, | |
| "size_gb": 1.3779706452041864, | |
| "path": "common_crawl/2022-05/en_middle_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00708-of-01024.jsonl", | |
| "size": 845898382, | |
| "size_mb": 806.7115612030029, | |
| "size_gb": 0.7878042589873075, | |
| "path": "c4/c4-train.00708-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00295-of-01024.jsonl", | |
| "size": 845931876, | |
| "size_mb": 806.7435035705566, | |
| "size_gb": 0.7878354527056217, | |
| "path": "c4/c4-train.00295-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00468-of-01024.jsonl", | |
| "size": 849622907, | |
| "size_mb": 810.2635450363159, | |
| "size_gb": 0.7912729931995273, | |
| "path": "c4/c4-train.00468-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00466-of-01024.jsonl", | |
| "size": 844998927, | |
| "size_mb": 805.8537740707397, | |
| "size_gb": 0.7869665762409568, | |
| "path": "c4/c4-train.00466-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00467-of-01024.jsonl", | |
| "size": 841737850, | |
| "size_mb": 802.7437686920166, | |
| "size_gb": 0.7839294616132975, | |
| "path": "c4/c4-train.00467-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00707-of-01024.jsonl", | |
| "size": 841613079, | |
| "size_mb": 802.6247777938843, | |
| "size_gb": 0.7838132595643401, | |
| "path": "c4/c4-train.00707-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00706-of-01024.jsonl", | |
| "size": 847874563, | |
| "size_mb": 808.596194267273, | |
| "size_gb": 0.7896447209641337, | |
| "path": "c4/c4-train.00706-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00709-of-01024.jsonl", | |
| "size": 843534859, | |
| "size_mb": 804.4575300216675, | |
| "size_gb": 0.7856030566617846, | |
| "path": "c4/c4-train.00709-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00705-of-01024.jsonl", | |
| "size": 847942499, | |
| "size_mb": 808.6609830856323, | |
| "size_gb": 0.7897079912945628, | |
| "path": "c4/c4-train.00705-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1572580401, | |
| "size_mb": 1499.7295389175415, | |
| "size_gb": 1.4645796278491616, | |
| "path": "common_crawl/2021-04/en_middle_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0099.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1589899728, | |
| "size_mb": 1516.2465362548828, | |
| "size_gb": 1.480709508061409, | |
| "path": "common_crawl/2021-04/en_middle_0099.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00230-of-01024.jsonl", | |
| "size": 851002994, | |
| "size_mb": 811.5796985626221, | |
| "size_gb": 0.7925582993775606, | |
| "path": "c4/c4-train.00230-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00233-of-01024.jsonl", | |
| "size": 846509139, | |
| "size_mb": 807.2940244674683, | |
| "size_gb": 0.788373070769012, | |
| "path": "c4/c4-train.00233-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00231-of-01024.jsonl", | |
| "size": 842796996, | |
| "size_mb": 803.753849029541, | |
| "size_gb": 0.7849158681929111, | |
| "path": "c4/c4-train.00231-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_5f2b3517159b426bb1a9e81ca189abcd.sampled.jsonl", | |
| "size": 2312114722, | |
| "size_mb": 2205.0044269561768, | |
| "size_gb": 2.1533246356993914, | |
| "path": "github/filtered_5f2b3517159b426bb1a9e81ca189abcd.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00237-of-01024.jsonl", | |
| "size": 844887847, | |
| "size_mb": 805.7478399276733, | |
| "size_gb": 0.7868631249293685, | |
| "path": "c4/c4-train.00237-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_5abf005e4e634f1dbfa8bd20b5687092.sampled.jsonl", | |
| "size": 2328362468, | |
| "size_mb": 2220.499485015869, | |
| "size_gb": 2.1684565283358097, | |
| "path": "github/filtered_5abf005e4e634f1dbfa8bd20b5687092.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00232-of-01024.jsonl", | |
| "size": 847556783, | |
| "size_mb": 808.2931356430054, | |
| "size_gb": 0.7893487652763724, | |
| "path": "c4/c4-train.00232-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_610e908cafaa4d53958de50ad700822a.sampled.jsonl", | |
| "size": 1583812571, | |
| "size_mb": 1510.4413709640503, | |
| "size_gb": 1.4750404013320804, | |
| "path": "github/filtered_610e908cafaa4d53958de50ad700822a.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00282-of-01024.jsonl", | |
| "size": 842793271, | |
| "size_mb": 803.7502965927124, | |
| "size_gb": 0.7849123990163207, | |
| "path": "c4/c4-train.00282-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0097.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1604391077, | |
| "size_mb": 1530.0665636062622, | |
| "size_gb": 1.4942056285217404, | |
| "path": "common_crawl/2021-04/en_middle_0097.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1289856814, | |
| "size_mb": 1230.1033153533936, | |
| "size_gb": 1.2012727688997984, | |
| "path": "common_crawl/2020-05/en_middle_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1402362158, | |
| "size_mb": 1337.3967723846436, | |
| "size_gb": 1.3060515355318785, | |
| "path": "common_crawl/2019-30/en_middle_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00234-of-01024.jsonl", | |
| "size": 847652272, | |
| "size_mb": 808.3842010498047, | |
| "size_gb": 0.7894376963376999, | |
| "path": "c4/c4-train.00234-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1278192923, | |
| "size_mb": 1218.9797620773315, | |
| "size_gb": 1.190409923903644, | |
| "path": "common_crawl/2020-05/en_middle_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_1982fb29-c4ed-4dd3-855c-666e63bc62d9.jsonl", | |
| "size": 964909862, | |
| "size_mb": 920.209753036499, | |
| "size_gb": 0.8986423369497061, | |
| "path": "arxiv/arxiv_1982fb29-c4ed-4dd3-855c-666e63bc62d9.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01002-of-01024.jsonl", | |
| "size": 842594835, | |
| "size_mb": 803.561053276062, | |
| "size_gb": 0.7847275910899043, | |
| "path": "c4/c4-train.01002-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00781-of-01024.jsonl", | |
| "size": 845356741, | |
| "size_mb": 806.1950120925903, | |
| "size_gb": 0.7872998164966702, | |
| "path": "c4/c4-train.00781-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_78a13a25258f4d24923702c07445e20e.sampled.jsonl", | |
| "size": 1648012302, | |
| "size_mb": 1571.6670055389404, | |
| "size_gb": 1.5348310600966215, | |
| "path": "github/filtered_78a13a25258f4d24923702c07445e20e.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0101.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1623156615, | |
| "size_mb": 1547.9627752304077, | |
| "size_gb": 1.511682397685945, | |
| "path": "common_crawl/2021-04/en_middle_0101.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00240-of-01024.jsonl", | |
| "size": 845304442, | |
| "size_mb": 806.1451358795166, | |
| "size_gb": 0.7872511092573404, | |
| "path": "c4/c4-train.00240-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00239-of-01024.jsonl", | |
| "size": 843573612, | |
| "size_mb": 804.4944877624512, | |
| "size_gb": 0.7856391482055187, | |
| "path": "c4/c4-train.00239-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_6353ab14cb8f4623a7d75678d9e7f44e.sampled.jsonl", | |
| "size": 2505461951, | |
| "size_mb": 2389.3947134017944, | |
| "size_gb": 2.33339327480644, | |
| "path": "github/filtered_6353ab14cb8f4623a7d75678d9e7f44e.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1310517909, | |
| "size_mb": 1249.8072710037231, | |
| "size_gb": 1.2205149130895734, | |
| "path": "common_crawl/2020-05/en_middle_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1407025872, | |
| "size_mb": 1341.8444366455078, | |
| "size_gb": 1.3103949576616287, | |
| "path": "common_crawl/2019-30/en_middle_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1391606263, | |
| "size_mb": 1327.1391515731812, | |
| "size_gb": 1.2960343277081847, | |
| "path": "common_crawl/2019-30/en_middle_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00238-of-01024.jsonl", | |
| "size": 844602758, | |
| "size_mb": 805.4759578704834, | |
| "size_gb": 0.7865976151078939, | |
| "path": "c4/c4-train.00238-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00248-of-01024.jsonl", | |
| "size": 846414696, | |
| "size_mb": 807.2039566040039, | |
| "size_gb": 0.7882851138710976, | |
| "path": "c4/c4-train.00248-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00242-of-01024.jsonl", | |
| "size": 849257591, | |
| "size_mb": 809.9151525497437, | |
| "size_gb": 0.790932766161859, | |
| "path": "c4/c4-train.00242-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1594888077, | |
| "size_mb": 1521.0037965774536, | |
| "size_gb": 1.4853552700951695, | |
| "path": "common_crawl/2021-04/en_middle_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1569262546, | |
| "size_mb": 1496.5653858184814, | |
| "size_gb": 1.4614896345883608, | |
| "path": "common_crawl/2021-04/en_middle_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1804440161, | |
| "size_mb": 1720.8482370376587, | |
| "size_gb": 1.6805158564820886, | |
| "path": "common_crawl/2021-04/en_head_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1401819904, | |
| "size_mb": 1336.879638671875, | |
| "size_gb": 1.305546522140503, | |
| "path": "common_crawl/2019-30/en_middle_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_64747f25102740bab0ab54559569342a.sampled.jsonl", | |
| "size": 1838824922, | |
| "size_mb": 1753.640100479126, | |
| "size_gb": 1.7125391606241465, | |
| "path": "github/filtered_64747f25102740bab0ab54559569342a.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1349628922, | |
| "size_mb": 1287.106439590454, | |
| "size_gb": 1.2569398824125528, | |
| "path": "common_crawl/2020-05/en_middle_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1541643162, | |
| "size_mb": 1470.2254886627197, | |
| "size_gb": 1.4357670787721872, | |
| "path": "common_crawl/2022-05/en_middle_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1269749140, | |
| "size_mb": 1210.9271430969238, | |
| "size_gb": 1.1825460381805897, | |
| "path": "common_crawl/2020-05/en_middle_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1751470348, | |
| "size_mb": 1670.3322868347168, | |
| "size_gb": 1.6311838738620281, | |
| "path": "common_crawl/2021-04/en_middle_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0098.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1560539358, | |
| "size_mb": 1488.2463054656982, | |
| "size_gb": 1.453365532681346, | |
| "path": "common_crawl/2021-04/en_middle_0098.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1433214973, | |
| "size_mb": 1366.820309638977, | |
| "size_gb": 1.3347854586318135, | |
| "path": "common_crawl/2022-05/en_middle_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1570400051, | |
| "size_mb": 1497.6501951217651, | |
| "size_gb": 1.4625490186735988, | |
| "path": "common_crawl/2021-04/en_middle_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1525182876, | |
| "size_mb": 1454.5277366638184, | |
| "size_gb": 1.4204372428357601, | |
| "path": "common_crawl/2022-05/en_middle_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1593417974, | |
| "size_mb": 1519.6017971038818, | |
| "size_gb": 1.4839861299842596, | |
| "path": "common_crawl/2021-04/en_middle_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0076.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1536774630, | |
| "size_mb": 1465.5824947357178, | |
| "size_gb": 1.4312329050153494, | |
| "path": "common_crawl/2022-05/en_middle_0076.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1587825456, | |
| "size_mb": 1514.2683563232422, | |
| "size_gb": 1.4787776917219162, | |
| "path": "common_crawl/2021-04/en_middle_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1621067282, | |
| "size_mb": 1545.9702320098877, | |
| "size_gb": 1.509736554697156, | |
| "path": "common_crawl/2021-04/en_middle_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1540140751, | |
| "size_mb": 1468.7926778793335, | |
| "size_gb": 1.4343678494915366, | |
| "path": "common_crawl/2022-05/en_middle_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1296235486, | |
| "size_mb": 1236.1864910125732, | |
| "size_gb": 1.207213370129466, | |
| "path": "common_crawl/2020-05/en_middle_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1301581734, | |
| "size_mb": 1241.2850704193115, | |
| "size_gb": 1.212192451581359, | |
| "path": "common_crawl/2020-05/en_middle_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1533208826, | |
| "size_mb": 1462.181879043579, | |
| "size_gb": 1.4279119912534952, | |
| "path": "common_crawl/2022-05/en_middle_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1815647389, | |
| "size_mb": 1731.5362825393677, | |
| "size_gb": 1.6909534009173512, | |
| "path": "common_crawl/2021-04/en_head_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1519599897, | |
| "size_mb": 1449.203392982483, | |
| "size_gb": 1.415237688459456, | |
| "path": "common_crawl/2022-05/en_middle_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1526856539, | |
| "size_mb": 1456.1238660812378, | |
| "size_gb": 1.4219959629699588, | |
| "path": "common_crawl/2022-05/en_middle_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1487236250, | |
| "size_mb": 1418.3390140533447, | |
| "size_gb": 1.3850966934114695, | |
| "path": "common_crawl/2022-05/en_middle_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0070.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1427408523, | |
| "size_mb": 1361.28284740448, | |
| "size_gb": 1.3293777806684375, | |
| "path": "common_crawl/2022-05/en_middle_0070.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1606405525, | |
| "size_mb": 1531.9876909255981, | |
| "size_gb": 1.4960817294195294, | |
| "path": "common_crawl/2021-04/en_middle_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1585929440, | |
| "size_mb": 1512.4601745605469, | |
| "size_gb": 1.477011889219284, | |
| "path": "common_crawl/2021-04/en_middle_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0100.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1546927709, | |
| "size_mb": 1475.2652254104614, | |
| "size_gb": 1.4406886966899037, | |
| "path": "common_crawl/2021-04/en_middle_0100.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1517900555, | |
| "size_mb": 1447.5827741622925, | |
| "size_gb": 1.4136550528928638, | |
| "path": "common_crawl/2022-05/en_middle_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1601345055, | |
| "size_mb": 1527.1616506576538, | |
| "size_gb": 1.491368799470365, | |
| "path": "common_crawl/2021-04/en_middle_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1688947327, | |
| "size_mb": 1610.7056875228882, | |
| "size_gb": 1.5729547729715705, | |
| "path": "common_crawl/2021-04/en_middle_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1524315998, | |
| "size_mb": 1453.7010173797607, | |
| "size_gb": 1.4196298997849226, | |
| "path": "common_crawl/2022-05/en_middle_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1871546825, | |
| "size_mb": 1784.8461389541626, | |
| "size_gb": 1.7430138075724244, | |
| "path": "common_crawl/2021-04/en_head_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1507077162, | |
| "size_mb": 1437.2607822418213, | |
| "size_gb": 1.4035749826580286, | |
| "path": "common_crawl/2022-05/en_middle_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1450064011, | |
| "size_mb": 1382.88880443573, | |
| "size_gb": 1.3504773480817676, | |
| "path": "common_crawl/2019-30/en_middle_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1403730235, | |
| "size_mb": 1338.7014722824097, | |
| "size_gb": 1.3073256565257907, | |
| "path": "common_crawl/2022-05/en_middle_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1505675746, | |
| "size_mb": 1435.9242877960205, | |
| "size_gb": 1.4022698123008013, | |
| "path": "common_crawl/2022-05/en_middle_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1869154718, | |
| "size_mb": 1782.564847946167, | |
| "size_gb": 1.7407859843224287, | |
| "path": "common_crawl/2021-04/en_head_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1412317582, | |
| "size_mb": 1346.891004562378, | |
| "size_gb": 1.3153232466429472, | |
| "path": "common_crawl/2022-05/en_middle_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00761-of-01024.jsonl", | |
| "size": 848136441, | |
| "size_mb": 808.8459405899048, | |
| "size_gb": 0.7898886138573289, | |
| "path": "c4/c4-train.00761-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_400748d3-0076-4a04-8a1c-6055ba0b5a2d.jsonl", | |
| "size": 916832910, | |
| "size_mb": 874.3599987030029, | |
| "size_gb": 0.8538671862334013, | |
| "path": "arxiv/arxiv_400748d3-0076-4a04-8a1c-6055ba0b5a2d.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01001-of-01024.jsonl", | |
| "size": 844919100, | |
| "size_mb": 805.777645111084, | |
| "size_gb": 0.786892231553793, | |
| "path": "c4/c4-train.01001-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01000-of-01024.jsonl", | |
| "size": 842246777, | |
| "size_mb": 803.2291193008423, | |
| "size_gb": 0.7844034368172288, | |
| "path": "c4/c4-train.01000-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1313871072, | |
| "size_mb": 1253.0050964355469, | |
| "size_gb": 1.2236377894878387, | |
| "path": "common_crawl/2020-05/en_middle_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00280-of-01024.jsonl", | |
| "size": 845581847, | |
| "size_mb": 806.4096899032593, | |
| "size_gb": 0.7875094627961516, | |
| "path": "c4/c4-train.00280-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00281-of-01024.jsonl", | |
| "size": 844451290, | |
| "size_mb": 805.331506729126, | |
| "size_gb": 0.7864565495401621, | |
| "path": "c4/c4-train.00281-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_92ec488a-287d-4bf0-977b-6998cf0cf476.jsonl", | |
| "size": 897586630, | |
| "size_mb": 856.0053157806396, | |
| "size_gb": 0.8359426911920309, | |
| "path": "arxiv/arxiv_92ec488a-287d-4bf0-977b-6998cf0cf476.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1475886314, | |
| "size_mb": 1407.51487159729, | |
| "size_gb": 1.3745262417942286, | |
| "path": "common_crawl/2022-05/en_middle_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00760-of-01024.jsonl", | |
| "size": 847284287, | |
| "size_mb": 808.0332632064819, | |
| "size_gb": 0.78909498360008, | |
| "path": "c4/c4-train.00760-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00762-of-01024.jsonl", | |
| "size": 842893892, | |
| "size_mb": 803.8462562561035, | |
| "size_gb": 0.7850061096251011, | |
| "path": "c4/c4-train.00762-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_44e19375-3995-4dff-a3b6-8a25247a165c.jsonl", | |
| "size": 848586496, | |
| "size_mb": 809.275146484375, | |
| "size_gb": 0.7903077602386475, | |
| "path": "arxiv/arxiv_44e19375-3995-4dff-a3b6-8a25247a165c.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_c6cfd16905814d7c955df1f4754a8b11.sampled.jsonl", | |
| "size": 1943730484, | |
| "size_mb": 1853.6858406066895, | |
| "size_gb": 1.8102400787174702, | |
| "path": "github/filtered_c6cfd16905814d7c955df1f4754a8b11.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1855090687, | |
| "size_mb": 1769.1523427963257, | |
| "size_gb": 1.7276878347620368, | |
| "path": "common_crawl/2021-04/en_head_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2039310449, | |
| "size_mb": 1944.8379983901978, | |
| "size_gb": 1.8992558578029275, | |
| "path": "common_crawl/2020-05/en_head_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1391426735, | |
| "size_mb": 1326.9679403305054, | |
| "size_gb": 1.2958671292290092, | |
| "path": "common_crawl/2019-30/en_middle_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_bfbcad9633f04601ba2f824d082eaacf.sampled.jsonl", | |
| "size": 2265251880, | |
| "size_mb": 2160.3125381469727, | |
| "size_gb": 2.109680213034153, | |
| "path": "github/filtered_bfbcad9633f04601ba2f824d082eaacf.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_c82c6775f0b74dbdae4524bb9aebf0ef.sampled.jsonl", | |
| "size": 1813022210, | |
| "size_mb": 1729.0327167510986, | |
| "size_gb": 1.6885085124522448, | |
| "path": "github/filtered_c82c6775f0b74dbdae4524bb9aebf0ef.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00247-of-01024.jsonl", | |
| "size": 847517520, | |
| "size_mb": 808.2556915283203, | |
| "size_gb": 0.7893121987581253, | |
| "path": "c4/c4-train.00247-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00253-of-01024.jsonl", | |
| "size": 843464067, | |
| "size_mb": 804.3900175094604, | |
| "size_gb": 0.7855371264740825, | |
| "path": "c4/c4-train.00253-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00252-of-01024.jsonl", | |
| "size": 846398450, | |
| "size_mb": 807.1884632110596, | |
| "size_gb": 0.7882699836045504, | |
| "path": "c4/c4-train.00252-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_a777da5620f1467f8df3616b17d533dc.sampled.jsonl", | |
| "size": 1769794375, | |
| "size_mb": 1687.8074407577515, | |
| "size_gb": 1.6482494538649917, | |
| "path": "github/filtered_a777da5620f1467f8df3616b17d533dc.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1492886133, | |
| "size_mb": 1423.727162361145, | |
| "size_gb": 1.3903585569933057, | |
| "path": "common_crawl/2022-05/en_middle_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00251-of-01024.jsonl", | |
| "size": 845715563, | |
| "size_mb": 806.5372114181519, | |
| "size_gb": 0.7876339955255389, | |
| "path": "c4/c4-train.00251-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_677e10f9b0af4c489e60670352e7e224.sampled.jsonl", | |
| "size": 1679854462, | |
| "size_mb": 1602.0340557098389, | |
| "size_gb": 1.5644863825291395, | |
| "path": "github/filtered_677e10f9b0af4c489e60670352e7e224.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00802-of-01024.jsonl", | |
| "size": 845535990, | |
| "size_mb": 806.3659572601318, | |
| "size_gb": 0.7874667551368475, | |
| "path": "c4/c4-train.00802-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0080.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1479549920, | |
| "size_mb": 1411.0087585449219, | |
| "size_gb": 1.3779382407665253, | |
| "path": "common_crawl/2022-05/en_middle_0080.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_4a8cf52f-81d0-4875-9528-466b1cbc71e1.jsonl", | |
| "size": 966799975, | |
| "size_mb": 922.0123052597046, | |
| "size_gb": 0.9004026418551803, | |
| "path": "arxiv/arxiv_4a8cf52f-81d0-4875-9528-466b1cbc71e1.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00243-of-01024.jsonl", | |
| "size": 847543348, | |
| "size_mb": 808.2803230285645, | |
| "size_gb": 0.7893362529575825, | |
| "path": "c4/c4-train.00243-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2018960289, | |
| "size_mb": 1925.43057346344, | |
| "size_gb": 1.8803032943978906, | |
| "path": "common_crawl/2020-05/en_head_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0080.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1580712786, | |
| "size_mb": 1507.485185623169, | |
| "size_gb": 1.472153501585126, | |
| "path": "common_crawl/2021-04/en_middle_0080.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1436657400, | |
| "size_mb": 1370.1032638549805, | |
| "size_gb": 1.3379914686083794, | |
| "path": "common_crawl/2019-30/en_middle_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2059744523, | |
| "size_mb": 1964.3254499435425, | |
| "size_gb": 1.9182865722104907, | |
| "path": "common_crawl/2020-05/en_head_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1593824728, | |
| "size_mb": 1519.9897079467773, | |
| "size_gb": 1.4843649491667747, | |
| "path": "common_crawl/2021-04/en_middle_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1639153736, | |
| "size_mb": 1563.2188186645508, | |
| "size_gb": 1.5265808776021004, | |
| "path": "common_crawl/2021-04/en_middle_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0072.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 847408741, | |
| "size_mb": 808.151951789856, | |
| "size_gb": 0.7892108904197812, | |
| "path": "common_crawl/2021-04/en_head_0072.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1286379852, | |
| "size_mb": 1226.787425994873, | |
| "size_gb": 1.1980345956981182, | |
| "path": "common_crawl/2020-05/en_middle_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1601531925, | |
| "size_mb": 1527.3398637771606, | |
| "size_gb": 1.4915428357198834, | |
| "path": "common_crawl/2021-04/en_middle_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1543315833, | |
| "size_mb": 1471.8206720352173, | |
| "size_gb": 1.4373248750343919, | |
| "path": "common_crawl/2022-05/en_middle_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00255-of-01024.jsonl", | |
| "size": 847567591, | |
| "size_mb": 808.3034429550171, | |
| "size_gb": 0.7893588310107589, | |
| "path": "c4/c4-train.00255-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_68534e6a093744fa9f38fa1a9cf51232.sampled.jsonl", | |
| "size": 2127052577, | |
| "size_mb": 2028.5154123306274, | |
| "size_gb": 1.9809720823541284, | |
| "path": "github/filtered_68534e6a093744fa9f38fa1a9cf51232.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00245-of-01024.jsonl", | |
| "size": 846704517, | |
| "size_mb": 807.4803514480591, | |
| "size_gb": 0.7885550307109952, | |
| "path": "c4/c4-train.00245-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_7fa232e63a5d44a88a267181e9ac47b4.sampled.jsonl", | |
| "size": 8643466145, | |
| "size_mb": 8243.05166721344, | |
| "size_gb": 8.049855143763125, | |
| "path": "github/filtered_7fa232e63a5d44a88a267181e9ac47b4.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_791ed086a57f4879bb1596bed6d37bb3.sampled.jsonl", | |
| "size": 3062559007, | |
| "size_mb": 2920.683867454529, | |
| "size_gb": 2.8522303393110633, | |
| "path": "github/filtered_791ed086a57f4879bb1596bed6d37bb3.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_6adcf92fb2ee48059cb60579f2e931f7.sampled.jsonl", | |
| "size": 2736380830, | |
| "size_mb": 2609.616117477417, | |
| "size_gb": 2.54845323972404, | |
| "path": "github/filtered_6adcf92fb2ee48059cb60579f2e931f7.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_7b5cc18857a34a0981b54f082de55cf8.sampled.jsonl", | |
| "size": 2347026811, | |
| "size_mb": 2238.299189567566, | |
| "size_gb": 2.185839052312076, | |
| "path": "github/filtered_7b5cc18857a34a0981b54f082de55cf8.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_82a9a647b0eb4eb080d7ac15a13c765b.sampled.jsonl", | |
| "size": 1690856375, | |
| "size_mb": 1612.526297569275, | |
| "size_gb": 1.574732712469995, | |
| "path": "github/filtered_82a9a647b0eb4eb080d7ac15a13c765b.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_8bc3c4fae78c41d999b2ae6d97cce96c.sampled.jsonl", | |
| "size": 2126183071, | |
| "size_mb": 2027.6861867904663, | |
| "size_gb": 1.9801622917875648, | |
| "path": "github/filtered_8bc3c4fae78c41d999b2ae6d97cce96c.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1632690307, | |
| "size_mb": 1557.0548124313354, | |
| "size_gb": 1.520561340264976, | |
| "path": "common_crawl/2021-04/en_middle_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_7462f1a34f594e9f8334d9a0cbbf80e7.sampled.jsonl", | |
| "size": 2386490891, | |
| "size_mb": 2275.9350690841675, | |
| "size_gb": 2.2225928409025073, | |
| "path": "github/filtered_7462f1a34f594e9f8334d9a0cbbf80e7.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00249-of-01024.jsonl", | |
| "size": 843770932, | |
| "size_mb": 804.6826667785645, | |
| "size_gb": 0.7858229167759418, | |
| "path": "c4/c4-train.00249-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00244-of-01024.jsonl", | |
| "size": 845293855, | |
| "size_mb": 806.1350393295288, | |
| "size_gb": 0.787241249345243, | |
| "path": "c4/c4-train.00244-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00256-of-01024.jsonl", | |
| "size": 849422593, | |
| "size_mb": 810.0725107192993, | |
| "size_gb": 0.7910864362493157, | |
| "path": "c4/c4-train.00256-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1443835899, | |
| "size_mb": 1376.9492139816284, | |
| "size_gb": 1.344676966778934, | |
| "path": "common_crawl/2022-05/en_middle_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_81138c01f3a84f7fa8b56bf3d8fa35ce.sampled.jsonl", | |
| "size": 1849561085, | |
| "size_mb": 1763.878903388977, | |
| "size_gb": 1.722537991590798, | |
| "path": "github/filtered_81138c01f3a84f7fa8b56bf3d8fa35ce.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1580911500, | |
| "size_mb": 1507.6746940612793, | |
| "size_gb": 1.472338568419218, | |
| "path": "common_crawl/2021-04/en_middle_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00267-of-01024.jsonl", | |
| "size": 845303307, | |
| "size_mb": 806.1440534591675, | |
| "size_gb": 0.7872500522062182, | |
| "path": "c4/c4-train.00267-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_835ac26c7a70447b97f4ec38bcb969ed.sampled.jsonl", | |
| "size": 2900037751, | |
| "size_mb": 2765.6915197372437, | |
| "size_gb": 2.700870624743402, | |
| "path": "github/filtered_835ac26c7a70447b97f4ec38bcb969ed.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2108650723, | |
| "size_mb": 2010.9660367965698, | |
| "size_gb": 1.9638340203091502, | |
| "path": "common_crawl/2019-30/en_head_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_78e3afc1cfee41fbb7eaae2e5bfaa17b.sampled.jsonl", | |
| "size": 6033889874, | |
| "size_mb": 5754.365800857544, | |
| "size_gb": 5.619497852399945, | |
| "path": "github/filtered_78e3afc1cfee41fbb7eaae2e5bfaa17b.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00246-of-01024.jsonl", | |
| "size": 844151229, | |
| "size_mb": 805.0453462600708, | |
| "size_gb": 0.7861770959571004, | |
| "path": "c4/c4-train.00246-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1819796067, | |
| "size_mb": 1735.4927701950073, | |
| "size_gb": 1.6948171583935618, | |
| "path": "common_crawl/2021-04/en_head_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00254-of-01024.jsonl", | |
| "size": 844612130, | |
| "size_mb": 805.4848957061768, | |
| "size_gb": 0.7866063434630632, | |
| "path": "c4/c4-train.00254-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00250-of-01024.jsonl", | |
| "size": 843628659, | |
| "size_mb": 804.5469846725464, | |
| "size_gb": 0.7856904147192836, | |
| "path": "c4/c4-train.00250-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0071.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1538122685, | |
| "size_mb": 1466.8681001663208, | |
| "size_gb": 1.4324883790686727, | |
| "path": "common_crawl/2022-05/en_middle_0071.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1390389789, | |
| "size_mb": 1325.9790315628052, | |
| "size_gb": 1.294901398010552, | |
| "path": "common_crawl/2022-05/en_middle_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00258-of-01024.jsonl", | |
| "size": 842534885, | |
| "size_mb": 803.5038805007935, | |
| "size_gb": 0.7846717583015561, | |
| "path": "c4/c4-train.00258-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0081.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1434433541, | |
| "size_mb": 1367.9824266433716, | |
| "size_gb": 1.3359203385189176, | |
| "path": "common_crawl/2022-05/en_middle_0081.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2203807300, | |
| "size_mb": 2101.7144203186035, | |
| "size_gb": 2.0524554885923862, | |
| "path": "common_crawl/2023-06/en_head_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1570421208, | |
| "size_mb": 1497.6703720092773, | |
| "size_gb": 1.46256872266531, | |
| "path": "common_crawl/2021-04/en_middle_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_70f0aa43987643a7874286bca4faa66b.sampled.jsonl", | |
| "size": 2258323500, | |
| "size_mb": 2153.70512008667, | |
| "size_gb": 2.1032276563346386, | |
| "path": "github/filtered_70f0aa43987643a7874286bca4faa66b.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0078.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1469324595, | |
| "size_mb": 1401.2571287155151, | |
| "size_gb": 1.3684151647612453, | |
| "path": "common_crawl/2022-05/en_middle_0078.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1379057065, | |
| "size_mb": 1315.1713037490845, | |
| "size_gb": 1.2843469763174653, | |
| "path": "common_crawl/2019-30/en_middle_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1580611946, | |
| "size_mb": 1507.3890171051025, | |
| "size_gb": 1.4720595870167017, | |
| "path": "common_crawl/2021-04/en_middle_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0073.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1477319509, | |
| "size_mb": 1408.881672859192, | |
| "size_gb": 1.3758610086515546, | |
| "path": "common_crawl/2022-05/en_middle_0073.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0072.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1520543465, | |
| "size_mb": 1450.1032495498657, | |
| "size_gb": 1.4161164546385407, | |
| "path": "common_crawl/2022-05/en_middle_0072.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0086.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1364563540, | |
| "size_mb": 1301.3492012023926, | |
| "size_gb": 1.2708488292992115, | |
| "path": "common_crawl/2022-05/en_middle_0086.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1605648368, | |
| "size_mb": 1531.265609741211, | |
| "size_gb": 1.4953765720129013, | |
| "path": "common_crawl/2021-04/en_middle_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1568842259, | |
| "size_mb": 1496.164568901062, | |
| "size_gb": 1.4610982118174434, | |
| "path": "common_crawl/2021-04/en_middle_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0077.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1525182949, | |
| "size_mb": 1454.5278062820435, | |
| "size_gb": 1.420437310822308, | |
| "path": "common_crawl/2022-05/en_middle_0077.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1412575966, | |
| "size_mb": 1347.1374187469482, | |
| "size_gb": 1.3155638854950666, | |
| "path": "common_crawl/2022-05/en_middle_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0074.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1434486023, | |
| "size_mb": 1368.0324773788452, | |
| "size_gb": 1.3359692161902785, | |
| "path": "common_crawl/2022-05/en_middle_0074.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1576651727, | |
| "size_mb": 1503.6122579574585, | |
| "size_gb": 1.4683713456615806, | |
| "path": "common_crawl/2021-04/en_middle_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1614352440, | |
| "size_mb": 1539.5664596557617, | |
| "size_gb": 1.5034828707575798, | |
| "path": "common_crawl/2021-04/en_middle_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1746263271, | |
| "size_mb": 1665.366431236267, | |
| "size_gb": 1.626334405504167, | |
| "path": "common_crawl/2021-04/en_middle_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1590413625, | |
| "size_mb": 1516.736626625061, | |
| "size_gb": 1.4811881119385362, | |
| "path": "common_crawl/2021-04/en_middle_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1570989710, | |
| "size_mb": 1498.212537765503, | |
| "size_gb": 1.463098181411624, | |
| "path": "common_crawl/2021-04/en_middle_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0071.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1852037141, | |
| "size_mb": 1766.2402544021606, | |
| "size_gb": 1.72484399843961, | |
| "path": "common_crawl/2021-04/en_head_0071.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1599711590, | |
| "size_mb": 1525.6038570404053, | |
| "size_gb": 1.4898475166410208, | |
| "path": "common_crawl/2021-04/en_middle_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00323-of-01024.jsonl", | |
| "size": 847903878, | |
| "size_mb": 808.6241512298584, | |
| "size_gb": 0.7896720226854086, | |
| "path": "c4/c4-train.00323-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0079.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1467691526, | |
| "size_mb": 1399.699712753296, | |
| "size_gb": 1.3668942507356405, | |
| "path": "common_crawl/2022-05/en_middle_0079.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00801-of-01024.jsonl", | |
| "size": 844785781, | |
| "size_mb": 805.650502204895, | |
| "size_gb": 0.7867680685594678, | |
| "path": "c4/c4-train.00801-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_1958455d-6543-4307-a081-d86ce0637f9a.jsonl", | |
| "size": 798939537, | |
| "size_mb": 761.9281167984009, | |
| "size_gb": 0.7440704265609384, | |
| "path": "arxiv/arxiv_1958455d-6543-4307-a081-d86ce0637f9a.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1446279004, | |
| "size_mb": 1379.279140472412, | |
| "size_gb": 1.34695228561759, | |
| "path": "common_crawl/2022-05/en_middle_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1600372665, | |
| "size_mb": 1526.2343072891235, | |
| "size_gb": 1.4904631907120347, | |
| "path": "common_crawl/2021-04/en_middle_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1598445877, | |
| "size_mb": 1524.3967790603638, | |
| "size_gb": 1.4886687295511365, | |
| "path": "common_crawl/2021-04/en_middle_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_8dc430f8f7114f018440c7b3d990e602.sampled.jsonl", | |
| "size": 1542306538, | |
| "size_mb": 1470.85813331604, | |
| "size_gb": 1.4363848958164454, | |
| "path": "github/filtered_8dc430f8f7114f018440c7b3d990e602.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1515673640, | |
| "size_mb": 1445.4590225219727, | |
| "size_gb": 1.411581076681614, | |
| "path": "common_crawl/2022-05/en_middle_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1809475376, | |
| "size_mb": 1725.6501922607422, | |
| "size_gb": 1.685205265879631, | |
| "path": "common_crawl/2021-04/en_head_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0084.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1455891739, | |
| "size_mb": 1388.4465589523315, | |
| "size_gb": 1.3559048427268863, | |
| "path": "common_crawl/2022-05/en_middle_0084.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1575796396, | |
| "size_mb": 1502.7965507507324, | |
| "size_gb": 1.4675747565925121, | |
| "path": "common_crawl/2021-04/en_middle_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1604589544, | |
| "size_mb": 1530.2558364868164, | |
| "size_gb": 1.4943904653191566, | |
| "path": "common_crawl/2021-04/en_middle_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1576036208, | |
| "size_mb": 1503.0252532958984, | |
| "size_gb": 1.4677980989217758, | |
| "path": "common_crawl/2021-04/en_middle_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0070.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1847596635, | |
| "size_mb": 1762.0054578781128, | |
| "size_gb": 1.7207084549590945, | |
| "path": "common_crawl/2021-04/en_head_0070.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1635789096, | |
| "size_mb": 1560.0100479125977, | |
| "size_gb": 1.5234473124146461, | |
| "path": "common_crawl/2021-04/en_middle_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1608424152, | |
| "size_mb": 1533.9128036499023, | |
| "size_gb": 1.4979617223143578, | |
| "path": "common_crawl/2021-04/en_middle_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2114560738, | |
| "size_mb": 2016.6022663116455, | |
| "size_gb": 1.9693381506949663, | |
| "path": "common_crawl/2019-30/en_head_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1522386896, | |
| "size_mb": 1451.8612823486328, | |
| "size_gb": 1.4178332835435867, | |
| "path": "common_crawl/2022-05/en_middle_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0083.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1508326312, | |
| "size_mb": 1438.4520645141602, | |
| "size_gb": 1.4047383442521095, | |
| "path": "common_crawl/2022-05/en_middle_0083.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1568579873, | |
| "size_mb": 1495.9143381118774, | |
| "size_gb": 1.4608538458123803, | |
| "path": "common_crawl/2021-04/en_middle_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0075.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1518078080, | |
| "size_mb": 1447.7520751953125, | |
| "size_gb": 1.4138203859329224, | |
| "path": "common_crawl/2022-05/en_middle_0075.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0082.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1419796873, | |
| "size_mb": 1354.0238122940063, | |
| "size_gb": 1.3222888791933656, | |
| "path": "common_crawl/2019-30/en_middle_0082.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1510028569, | |
| "size_mb": 1440.075463294983, | |
| "size_gb": 1.4063236946240067, | |
| "path": "common_crawl/2022-05/en_middle_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1569031875, | |
| "size_mb": 1496.3454008102417, | |
| "size_gb": 1.4612748054787517, | |
| "path": "common_crawl/2022-05/en_middle_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2169657174, | |
| "size_mb": 2069.146322250366, | |
| "size_gb": 2.0206507053226233, | |
| "path": "common_crawl/2019-30/en_head_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1462758990, | |
| "size_mb": 1394.9956798553467, | |
| "size_gb": 1.362300468608737, | |
| "path": "common_crawl/2023-06/en_middle_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1601560729, | |
| "size_mb": 1527.3673334121704, | |
| "size_gb": 1.4915696615353227, | |
| "path": "common_crawl/2021-04/en_middle_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1495675094, | |
| "size_mb": 1426.3869228363037, | |
| "size_gb": 1.3929559793323278, | |
| "path": "common_crawl/2022-05/en_middle_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0082.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1385871442, | |
| "size_mb": 1321.670000076294, | |
| "size_gb": 1.2906933594495058, | |
| "path": "common_crawl/2022-05/en_middle_0082.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1441127206, | |
| "size_mb": 1374.366003036499, | |
| "size_gb": 1.342154299840331, | |
| "path": "common_crawl/2022-05/en_middle_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1555133361, | |
| "size_mb": 1483.090744972229, | |
| "size_gb": 1.4483308056369424, | |
| "path": "common_crawl/2022-05/en_middle_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1601550774, | |
| "size_mb": 1527.3578395843506, | |
| "size_gb": 1.4915603902190924, | |
| "path": "common_crawl/2021-04/en_middle_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0081.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1380275437, | |
| "size_mb": 1316.333233833313, | |
| "size_gb": 1.2854816736653447, | |
| "path": "common_crawl/2019-30/en_middle_0081.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1492442082, | |
| "size_mb": 1423.3036823272705, | |
| "size_gb": 1.389945002272725, | |
| "path": "common_crawl/2022-05/en_middle_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0085.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1343619570, | |
| "size_mb": 1281.3754749298096, | |
| "size_gb": 1.2513432372361422, | |
| "path": "common_crawl/2022-05/en_middle_0085.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1540980913, | |
| "size_mb": 1469.593918800354, | |
| "size_gb": 1.4351503113284707, | |
| "path": "common_crawl/2021-04/en_middle_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1475024946, | |
| "size_mb": 1406.6934070587158, | |
| "size_gb": 1.3737240303307772, | |
| "path": "common_crawl/2022-05/en_middle_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00319-of-01024.jsonl", | |
| "size": 847918426, | |
| "size_mb": 808.6380252838135, | |
| "size_gb": 0.7896855715662241, | |
| "path": "c4/c4-train.00319-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00318-of-01024.jsonl", | |
| "size": 846179347, | |
| "size_mb": 806.979510307312, | |
| "size_gb": 0.7880659280344844, | |
| "path": "c4/c4-train.00318-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00799-of-01024.jsonl", | |
| "size": 845091890, | |
| "size_mb": 805.9424304962158, | |
| "size_gb": 0.7870531547814608, | |
| "path": "c4/c4-train.00799-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00800-of-01024.jsonl", | |
| "size": 838484970, | |
| "size_mb": 799.641580581665, | |
| "size_gb": 0.7808999810367823, | |
| "path": "c4/c4-train.00800-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00805-of-01024.jsonl", | |
| "size": 849322002, | |
| "size_mb": 809.9765796661377, | |
| "size_gb": 0.7909927535802126, | |
| "path": "c4/c4-train.00805-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00804-of-01024.jsonl", | |
| "size": 843328272, | |
| "size_mb": 804.2605133056641, | |
| "size_gb": 0.7854106575250626, | |
| "path": "c4/c4-train.00804-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00320-of-01024.jsonl", | |
| "size": 847713199, | |
| "size_mb": 808.4423055648804, | |
| "size_gb": 0.7894944390282035, | |
| "path": "c4/c4-train.00320-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_0de84cfc-c080-471f-b139-1bf061db4feb.jsonl", | |
| "size": 991098441, | |
| "size_mb": 945.1851282119751, | |
| "size_gb": 0.9230323517695069, | |
| "path": "arxiv/arxiv_0de84cfc-c080-471f-b139-1bf061db4feb.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1313458492, | |
| "size_mb": 1252.611629486084, | |
| "size_gb": 1.223253544420004, | |
| "path": "common_crawl/2020-05/en_middle_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00325-of-01024.jsonl", | |
| "size": 844104424, | |
| "size_mb": 805.0007095336914, | |
| "size_gb": 0.7861335054039955, | |
| "path": "c4/c4-train.00325-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1266252377, | |
| "size_mb": 1207.5923700332642, | |
| "size_gb": 1.1792894238606095, | |
| "path": "common_crawl/2020-05/en_middle_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_0fbdd8ad-32d8-4228-9a40-e09dde689760.jsonl", | |
| "size": 961753740, | |
| "size_mb": 917.1998405456543, | |
| "size_gb": 0.8957029692828655, | |
| "path": "arxiv/arxiv_0fbdd8ad-32d8-4228-9a40-e09dde689760.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0084.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1370716316, | |
| "size_mb": 1307.2169456481934, | |
| "size_gb": 1.2765790484845638, | |
| "path": "common_crawl/2019-30/en_middle_0084.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2168270976, | |
| "size_mb": 2067.8243408203125, | |
| "size_gb": 2.0193597078323364, | |
| "path": "common_crawl/2019-30/en_head_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_30440198-cd90-48c6-82c1-ea871b8c21c5.jsonl", | |
| "size": 965382473, | |
| "size_mb": 920.6604700088501, | |
| "size_gb": 0.8990824902430177, | |
| "path": "arxiv/arxiv_30440198-cd90-48c6-82c1-ea871b8c21c5.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_7c54b908a2df4ec2ba316d2081fc674e.sampled.jsonl", | |
| "size": 4035116763, | |
| "size_mb": 3848.1872205734253, | |
| "size_gb": 3.7579953325912356, | |
| "path": "github/filtered_7c54b908a2df4ec2ba316d2081fc674e.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_077ae8de-a68e-47e7-95a6-6d82f8f4eeb9.jsonl", | |
| "size": 890094980, | |
| "size_mb": 848.8607215881348, | |
| "size_gb": 0.8289655484259129, | |
| "path": "arxiv/arxiv_077ae8de-a68e-47e7-95a6-6d82f8f4eeb9.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00497-of-01024.jsonl", | |
| "size": 846911285, | |
| "size_mb": 807.6775407791138, | |
| "size_gb": 0.7887475984171033, | |
| "path": "c4/c4-train.00497-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00736-of-01024.jsonl", | |
| "size": 844558506, | |
| "size_mb": 805.4337558746338, | |
| "size_gb": 0.7865564022213221, | |
| "path": "c4/c4-train.00736-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00735-of-01024.jsonl", | |
| "size": 847678763, | |
| "size_mb": 808.4094648361206, | |
| "size_gb": 0.789462368004024, | |
| "path": "c4/c4-train.00735-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00259-of-01024.jsonl", | |
| "size": 845595193, | |
| "size_mb": 806.422417640686, | |
| "size_gb": 0.7875218922272325, | |
| "path": "c4/c4-train.00259-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1603054909, | |
| "size_mb": 1528.7922945022583, | |
| "size_gb": 1.4929612250998616, | |
| "path": "common_crawl/2021-04/en_middle_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00737-of-01024.jsonl", | |
| "size": 848711454, | |
| "size_mb": 809.3943157196045, | |
| "size_gb": 0.7904241364449263, | |
| "path": "c4/c4-train.00737-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00324-of-01024.jsonl", | |
| "size": 845784042, | |
| "size_mb": 806.602518081665, | |
| "size_gb": 0.787697771564126, | |
| "path": "c4/c4-train.00324-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0089.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1285279830, | |
| "size_mb": 1225.7383632659912, | |
| "size_gb": 1.1970101203769445, | |
| "path": "common_crawl/2020-05/en_middle_0089.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_7edd0af0b61c426e93e8bd3f549a8f78.sampled.jsonl", | |
| "size": 1827292438, | |
| "size_mb": 1742.64186668396, | |
| "size_gb": 1.7017986979335546, | |
| "path": "github/filtered_7edd0af0b61c426e93e8bd3f549a8f78.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_629fe3ca-075f-4663-9b81-b807f3b42bf2.jsonl", | |
| "size": 947361453, | |
| "size_mb": 903.4742860794067, | |
| "size_gb": 0.8822991074994206, | |
| "path": "arxiv/arxiv_629fe3ca-075f-4663-9b81-b807f3b42bf2.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00798-of-01024.jsonl", | |
| "size": 846883880, | |
| "size_mb": 807.6514053344727, | |
| "size_gb": 0.788722075521946, | |
| "path": "c4/c4-train.00798-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00150-of-01024.jsonl", | |
| "size": 844240391, | |
| "size_mb": 805.1303777694702, | |
| "size_gb": 0.7862601345404983, | |
| "path": "c4/c4-train.00150-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2199663345, | |
| "size_mb": 2097.7624368667603, | |
| "size_gb": 2.0485961297526956, | |
| "path": "common_crawl/2019-30/en_head_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0083.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1410789973, | |
| "size_mb": 1345.434163093567, | |
| "size_gb": 1.3139005498960614, | |
| "path": "common_crawl/2019-30/en_middle_0083.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1345786300, | |
| "size_mb": 1283.4418296813965, | |
| "size_gb": 1.2533611617982388, | |
| "path": "common_crawl/2020-05/en_middle_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0078.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1360816441, | |
| "size_mb": 1297.775689125061, | |
| "size_gb": 1.2673590714111924, | |
| "path": "common_crawl/2019-30/en_middle_0078.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0071.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1607970022, | |
| "size_mb": 1533.4797115325928, | |
| "size_gb": 1.4975387807935476, | |
| "path": "common_crawl/2021-04/en_middle_0071.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00065-of-01024.jsonl", | |
| "size": 847472374, | |
| "size_mb": 808.2126369476318, | |
| "size_gb": 0.7892701532691717, | |
| "path": "c4/c4-train.00065-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1570419016, | |
| "size_mb": 1497.6682815551758, | |
| "size_gb": 1.4625666812062263, | |
| "path": "common_crawl/2021-04/en_middle_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1309610499, | |
| "size_mb": 1248.941897392273, | |
| "size_gb": 1.2196698216721416, | |
| "path": "common_crawl/2020-05/en_middle_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1524550994, | |
| "size_mb": 1453.925127029419, | |
| "size_gb": 1.419848756864667, | |
| "path": "common_crawl/2022-05/en_middle_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2082903463, | |
| "size_mb": 1986.4115362167358, | |
| "size_gb": 1.939855015836656, | |
| "path": "common_crawl/2019-30/en_head_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0096.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 231227598, | |
| "size_mb": 220.51582145690918, | |
| "size_gb": 0.21534748189151287, | |
| "path": "common_crawl/2022-05/en_middle_0096.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_a3d3e6f3f7d5495ca9ecf94d808fd350.sampled.jsonl", | |
| "size": 1979210866, | |
| "size_mb": 1887.522569656372, | |
| "size_gb": 1.8432837594300508, | |
| "path": "github/filtered_a3d3e6f3f7d5495ca9ecf94d808fd350.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2104462875, | |
| "size_mb": 2006.9721937179565, | |
| "size_gb": 1.959933782927692, | |
| "path": "common_crawl/2022-05/en_head_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1426836375, | |
| "size_mb": 1360.7372045516968, | |
| "size_gb": 1.3288449263200164, | |
| "path": "common_crawl/2022-05/en_middle_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0070.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1639740787, | |
| "size_mb": 1563.7786741256714, | |
| "size_gb": 1.527127611450851, | |
| "path": "common_crawl/2021-04/en_middle_0070.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1361459785, | |
| "size_mb": 1298.389229774475, | |
| "size_gb": 1.2679582322016358, | |
| "path": "common_crawl/2020-05/en_middle_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2152876593, | |
| "size_mb": 2053.1431131362915, | |
| "size_gb": 2.0050225714221597, | |
| "path": "common_crawl/2022-05/en_head_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2171083343, | |
| "size_mb": 2070.506422996521, | |
| "size_gb": 2.02197892870754, | |
| "path": "common_crawl/2019-30/en_head_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1505219486, | |
| "size_mb": 1435.489164352417, | |
| "size_gb": 1.4018448870629072, | |
| "path": "common_crawl/2023-06/en_middle_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_a26c51ffe2924fd8ad6694c6aa0eacc5.sampled.jsonl", | |
| "size": 2249120831, | |
| "size_mb": 2144.928771018982, | |
| "size_gb": 2.0946570029482245, | |
| "path": "github/filtered_a26c51ffe2924fd8ad6694c6aa0eacc5.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0087.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1471005652, | |
| "size_mb": 1402.86030960083, | |
| "size_gb": 1.3699807710945606, | |
| "path": "common_crawl/2022-05/en_middle_0087.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0090.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1294180979, | |
| "size_mb": 1234.2271604537964, | |
| "size_gb": 1.2052999613806605, | |
| "path": "common_crawl/2020-05/en_middle_0090.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0091.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1520567273, | |
| "size_mb": 1450.1259546279907, | |
| "size_gb": 1.4161386275663972, | |
| "path": "common_crawl/2023-06/en_middle_0091.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_9eea501af8544d0b88f0b002850829d4.sampled.jsonl", | |
| "size": 1568824921, | |
| "size_mb": 1496.1480340957642, | |
| "size_gb": 1.4610820645466447, | |
| "path": "github/filtered_9eea501af8544d0b88f0b002850829d4.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00263-of-01024.jsonl", | |
| "size": 846470215, | |
| "size_mb": 807.2569036483765, | |
| "size_gb": 0.7883368199691176, | |
| "path": "c4/c4-train.00263-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_8e1db2cf6c98420a88bad52fd57f4aa7.sampled.jsonl", | |
| "size": 1781479933, | |
| "size_mb": 1698.951657295227, | |
| "size_gb": 1.6591324778273702, | |
| "path": "github/filtered_8e1db2cf6c98420a88bad52fd57f4aa7.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_96175b6e4c764abfbbf14e78d4fd6464.sampled.jsonl", | |
| "size": 1536611344, | |
| "size_mb": 1465.426773071289, | |
| "size_gb": 1.4310808330774307, | |
| "path": "github/filtered_96175b6e4c764abfbbf14e78d4fd6464.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_944beac1347f491faa88f43c25d26fe4.sampled.jsonl", | |
| "size": 1763318505, | |
| "size_mb": 1681.6315698623657, | |
| "size_gb": 1.6422183299437165, | |
| "path": "github/filtered_944beac1347f491faa88f43c25d26fe4.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_ca73c69896b34adbbe468d78d9f134bc.sampled.jsonl", | |
| "size": 1869634254, | |
| "size_mb": 1783.0221691131592, | |
| "size_gb": 1.7412325870245695, | |
| "path": "github/filtered_ca73c69896b34adbbe468d78d9f134bc.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00262-of-01024.jsonl", | |
| "size": 848191688, | |
| "size_mb": 808.8986282348633, | |
| "size_gb": 0.7899400666356087, | |
| "path": "c4/c4-train.00262-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00266-of-01024.jsonl", | |
| "size": 844331412, | |
| "size_mb": 805.2171821594238, | |
| "size_gb": 0.7863449044525623, | |
| "path": "c4/c4-train.00266-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00260-of-01024.jsonl", | |
| "size": 847397657, | |
| "size_mb": 808.1413812637329, | |
| "size_gb": 0.7892005676403642, | |
| "path": "c4/c4-train.00260-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_919156910e704e6ca52e0d4880cdbb63.sampled.jsonl", | |
| "size": 1620388542, | |
| "size_mb": 1545.3229351043701, | |
| "size_gb": 1.5091044288128614, | |
| "path": "github/filtered_919156910e704e6ca52e0d4880cdbb63.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00264-of-01024.jsonl", | |
| "size": 846162829, | |
| "size_mb": 806.9637575149536, | |
| "size_gb": 0.7880505444481969, | |
| "path": "c4/c4-train.00264-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00265-of-01024.jsonl", | |
| "size": 845001170, | |
| "size_mb": 805.8559131622314, | |
| "size_gb": 0.7869686651974916, | |
| "path": "c4/c4-train.00265-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00261-of-01024.jsonl", | |
| "size": 845140485, | |
| "size_mb": 805.9887742996216, | |
| "size_gb": 0.7870984124019742, | |
| "path": "c4/c4-train.00261-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_977d1fcdab92452d9dc38b2f4a99389b.sampled.jsonl", | |
| "size": 1905423582, | |
| "size_mb": 1817.1535320281982, | |
| "size_gb": 1.7745639961212873, | |
| "path": "github/filtered_977d1fcdab92452d9dc38b2f4a99389b.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_a3a01a2f81ed4cd2afb30e175200b48f.sampled.jsonl", | |
| "size": 1564503129, | |
| "size_mb": 1492.0264520645142, | |
| "size_gb": 1.457057082094252, | |
| "path": "github/filtered_a3a01a2f81ed4cd2afb30e175200b48f.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1369709098, | |
| "size_mb": 1306.2563877105713, | |
| "size_gb": 1.2756410036236048, | |
| "path": "common_crawl/2020-05/en_middle_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2155219501, | |
| "size_mb": 2055.3774843215942, | |
| "size_gb": 2.007204574532807, | |
| "path": "common_crawl/2019-30/en_head_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1614506367, | |
| "size_mb": 1539.7132558822632, | |
| "size_gb": 1.5036262264475226, | |
| "path": "common_crawl/2021-04/en_middle_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0091.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1473697825, | |
| "size_mb": 1405.4277658462524, | |
| "size_gb": 1.372488052584231, | |
| "path": "common_crawl/2022-05/en_middle_0091.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2211254812, | |
| "size_mb": 2108.816921234131, | |
| "size_gb": 2.059391524642706, | |
| "path": "common_crawl/2019-30/en_head_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1515241588, | |
| "size_mb": 1445.0469856262207, | |
| "size_gb": 1.4111786969006062, | |
| "path": "common_crawl/2022-05/en_middle_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1599806936, | |
| "size_mb": 1525.6947860717773, | |
| "size_gb": 1.48993631452322, | |
| "path": "common_crawl/2021-04/en_middle_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0076.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1580336550, | |
| "size_mb": 1507.1263790130615, | |
| "size_gb": 1.471803104504943, | |
| "path": "common_crawl/2021-04/en_middle_0076.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0085.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1357394165, | |
| "size_mb": 1294.5119524002075, | |
| "size_gb": 1.2641718285158277, | |
| "path": "common_crawl/2019-30/en_middle_0085.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1136175934, | |
| "size_mb": 1083.5418071746826, | |
| "size_gb": 1.058146296069026, | |
| "path": "common_crawl/2022-05/en_head_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0095.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1498677850, | |
| "size_mb": 1429.2505741119385, | |
| "size_gb": 1.39575251378119, | |
| "path": "common_crawl/2022-05/en_middle_0095.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0089.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1482353652, | |
| "size_mb": 1413.6826057434082, | |
| "size_gb": 1.380549419671297, | |
| "path": "common_crawl/2022-05/en_middle_0089.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_b56453718c5f46efa9c46feb194b0d6e.sampled.jsonl", | |
| "size": 2085290470, | |
| "size_mb": 1988.6879634857178, | |
| "size_gb": 1.9420780893415213, | |
| "path": "github/filtered_b56453718c5f46efa9c46feb194b0d6e.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2187943232, | |
| "size_mb": 2086.5852661132812, | |
| "size_gb": 2.037680923938751, | |
| "path": "common_crawl/2022-05/en_head_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00388-of-01024.jsonl", | |
| "size": 848164868, | |
| "size_mb": 808.8730506896973, | |
| "size_gb": 0.7899150885641575, | |
| "path": "c4/c4-train.00388-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0094.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1495974094, | |
| "size_mb": 1426.6720714569092, | |
| "size_gb": 1.3932344447821379, | |
| "path": "common_crawl/2022-05/en_middle_0094.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0077.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1594146504, | |
| "size_mb": 1520.2965774536133, | |
| "size_gb": 1.4846646264195442, | |
| "path": "common_crawl/2021-04/en_middle_0077.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1578248809, | |
| "size_mb": 1505.1353540420532, | |
| "size_gb": 1.4698587441816926, | |
| "path": "common_crawl/2021-04/en_middle_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1505583801, | |
| "size_mb": 1435.8366022109985, | |
| "size_gb": 1.4021841818466783, | |
| "path": "common_crawl/2022-05/en_middle_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00000-of-01024.jsonl", | |
| "size": 846322946, | |
| "size_mb": 807.1164569854736, | |
| "size_gb": 0.7881996650248766, | |
| "path": "c4/c4-train.00000-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00868-of-01024.jsonl", | |
| "size": 848779260, | |
| "size_mb": 809.4589805603027, | |
| "size_gb": 0.7904872857034206, | |
| "path": "c4/c4-train.00868-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00005-of-01024.jsonl", | |
| "size": 842771838, | |
| "size_mb": 803.7298564910889, | |
| "size_gb": 0.784892437979579, | |
| "path": "c4/c4-train.00005-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0080.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1380300654, | |
| "size_mb": 1316.3572826385498, | |
| "size_gb": 1.2855051588267088, | |
| "path": "common_crawl/2019-30/en_middle_0080.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_11c659c1-ffbf-4455-abfd-058f6bbf4bb2.jsonl", | |
| "size": 982198468, | |
| "size_mb": 936.697452545166, | |
| "size_gb": 0.9147436060011387, | |
| "path": "arxiv/arxiv_11c659c1-ffbf-4455-abfd-058f6bbf4bb2.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0079.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1373522203, | |
| "size_mb": 1309.8928480148315, | |
| "size_gb": 1.279192234389484, | |
| "path": "common_crawl/2019-30/en_middle_0079.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_11a1943edfa349c7939382799599eed6.sampled.jsonl", | |
| "size": 3670811474, | |
| "size_mb": 3500.758623123169, | |
| "size_gb": 3.4187095928937197, | |
| "path": "github/filtered_11a1943edfa349c7939382799599eed6.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0072.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1586864658, | |
| "size_mb": 1513.3520679473877, | |
| "size_gb": 1.4778828788548708, | |
| "path": "common_crawl/2021-04/en_middle_0072.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_17197bd2478044bebd9ff4634b6dfcee.sampled.jsonl", | |
| "size": 4292037006, | |
| "size_mb": 4093.205457687378, | |
| "size_gb": 3.99727095477283, | |
| "path": "github/filtered_17197bd2478044bebd9ff4634b6dfcee.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1484676182, | |
| "size_mb": 1415.8975429534912, | |
| "size_gb": 1.3827124442905188, | |
| "path": "common_crawl/2022-05/en_middle_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01004-of-01024.jsonl", | |
| "size": 846907510, | |
| "size_mb": 807.6739406585693, | |
| "size_gb": 0.7887440826743841, | |
| "path": "c4/c4-train.01004-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1593867437, | |
| "size_mb": 1520.0304384231567, | |
| "size_gb": 1.484404725022614, | |
| "path": "common_crawl/2021-04/en_middle_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00332-of-01024.jsonl", | |
| "size": 845228844, | |
| "size_mb": 806.0730400085449, | |
| "size_gb": 0.7871807031333447, | |
| "path": "c4/c4-train.00332-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1374771552, | |
| "size_mb": 1311.0843200683594, | |
| "size_gb": 1.2803557813167572, | |
| "path": "common_crawl/2019-30/en_middle_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_b6225e159b86432d9fa5bf226bb51393.sampled.jsonl", | |
| "size": 5169910541, | |
| "size_mb": 4930.410901069641, | |
| "size_gb": 4.814854395575821, | |
| "path": "github/filtered_b6225e159b86432d9fa5bf226bb51393.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0078.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1628126477, | |
| "size_mb": 1552.702404975891, | |
| "size_gb": 1.5163109423592687, | |
| "path": "common_crawl/2021-04/en_middle_0078.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1636558097, | |
| "size_mb": 1560.7434244155884, | |
| "size_gb": 1.524163500405848, | |
| "path": "common_crawl/2021-04/en_middle_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00764-of-01024.jsonl", | |
| "size": 845950720, | |
| "size_mb": 806.761474609375, | |
| "size_gb": 0.7878530025482178, | |
| "path": "c4/c4-train.00764-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1516532161, | |
| "size_mb": 1446.277771949768, | |
| "size_gb": 1.4123806366696954, | |
| "path": "common_crawl/2022-05/en_middle_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00812-of-01024.jsonl", | |
| "size": 848474555, | |
| "size_mb": 809.1683912277222, | |
| "size_gb": 0.7902035070583224, | |
| "path": "c4/c4-train.00812-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00867-of-01024.jsonl", | |
| "size": 846190379, | |
| "size_mb": 806.9900312423706, | |
| "size_gb": 0.7880762023851275, | |
| "path": "c4/c4-train.00867-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_75af5d17-5ebb-4460-9f2a-dc9fe880a936.jsonl", | |
| "size": 895020782, | |
| "size_mb": 853.5583324432373, | |
| "size_gb": 0.8335530590265989, | |
| "path": "arxiv/arxiv_75af5d17-5ebb-4460-9f2a-dc9fe880a936.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_f50efdc6-f88e-4fa6-9ef6-dd1d8314bb36.jsonl", | |
| "size": 897833878, | |
| "size_mb": 856.2411098480225, | |
| "size_gb": 0.8361729588359594, | |
| "path": "arxiv/arxiv_f50efdc6-f88e-4fa6-9ef6-dd1d8314bb36.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1841500713, | |
| "size_mb": 1756.191933631897, | |
| "size_gb": 1.7150311851873994, | |
| "path": "common_crawl/2021-04/en_head_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00284-of-01024.jsonl", | |
| "size": 845494278, | |
| "size_mb": 806.3261775970459, | |
| "size_gb": 0.7874279078096151, | |
| "path": "c4/c4-train.00284-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1478298818, | |
| "size_mb": 1409.8156147003174, | |
| "size_gb": 1.3767730612307787, | |
| "path": "common_crawl/2023-06/en_middle_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01019-of-01024.jsonl", | |
| "size": 849434473, | |
| "size_mb": 810.0838403701782, | |
| "size_gb": 0.7910975003615022, | |
| "path": "c4/c4-train.01019-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1437399937, | |
| "size_mb": 1370.8114023208618, | |
| "size_gb": 1.3386830100789666, | |
| "path": "common_crawl/2019-30/en_middle_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_872b620a-b4fd-45d3-92bc-ff0584447705.jsonl", | |
| "size": 983831016, | |
| "size_mb": 938.2543716430664, | |
| "size_gb": 0.916264034807682, | |
| "path": "arxiv/arxiv_872b620a-b4fd-45d3-92bc-ff0584447705.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2198693335, | |
| "size_mb": 2096.837363243103, | |
| "size_gb": 2.047692737542093, | |
| "path": "common_crawl/2023-06/en_head_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0075.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1582943578, | |
| "size_mb": 1509.6126346588135, | |
| "size_gb": 1.4742310885339975, | |
| "path": "common_crawl/2021-04/en_middle_0075.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0092.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1478907846, | |
| "size_mb": 1410.3964290618896, | |
| "size_gb": 1.3773402627557516, | |
| "path": "common_crawl/2022-05/en_middle_0092.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0074.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1584368226, | |
| "size_mb": 1510.971284866333, | |
| "size_gb": 1.4755578953772783, | |
| "path": "common_crawl/2021-04/en_middle_0074.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0088.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1452115142, | |
| "size_mb": 1384.8449153900146, | |
| "size_gb": 1.3523876126855612, | |
| "path": "common_crawl/2022-05/en_middle_0088.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1600139724, | |
| "size_mb": 1526.0121574401855, | |
| "size_gb": 1.4902462475001812, | |
| "path": "common_crawl/2021-04/en_middle_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0073.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1558583575, | |
| "size_mb": 1486.3811254501343, | |
| "size_gb": 1.4515440678223968, | |
| "path": "common_crawl/2021-04/en_middle_0073.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0093.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1508162572, | |
| "size_mb": 1438.2959098815918, | |
| "size_gb": 1.404585849493742, | |
| "path": "common_crawl/2022-05/en_middle_0093.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2006186259, | |
| "size_mb": 1913.248309135437, | |
| "size_gb": 1.8684065518900752, | |
| "path": "common_crawl/2020-05/en_head_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0079.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1591235082, | |
| "size_mb": 1517.5200290679932, | |
| "size_gb": 1.481953153386712, | |
| "path": "common_crawl/2021-04/en_middle_0079.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2111009186, | |
| "size_mb": 2013.2152423858643, | |
| "size_gb": 1.9660305101424456, | |
| "path": "common_crawl/2022-05/en_head_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1629959967, | |
| "size_mb": 1554.4509572982788, | |
| "size_gb": 1.5180185129866004, | |
| "path": "common_crawl/2021-04/en_middle_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1502405943, | |
| "size_mb": 1432.8059606552124, | |
| "size_gb": 1.3992245709523559, | |
| "path": "common_crawl/2022-05/en_middle_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00330-of-01024.jsonl", | |
| "size": 846253404, | |
| "size_mb": 807.0501365661621, | |
| "size_gb": 0.7881348989903927, | |
| "path": "c4/c4-train.00330-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00268-of-01024.jsonl", | |
| "size": 842682819, | |
| "size_mb": 803.6449613571167, | |
| "size_gb": 0.7848095325753093, | |
| "path": "c4/c4-train.00268-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2180708970, | |
| "size_mb": 2079.6861362457275, | |
| "size_gb": 2.0309434924274683, | |
| "path": "common_crawl/2022-05/en_head_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00334-of-01024.jsonl", | |
| "size": 843386300, | |
| "size_mb": 804.3158531188965, | |
| "size_gb": 0.7854647003114223, | |
| "path": "c4/c4-train.00334-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0090.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1478304001, | |
| "size_mb": 1409.8205575942993, | |
| "size_gb": 1.376777888275683, | |
| "path": "common_crawl/2022-05/en_middle_0090.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00817-of-01024.jsonl", | |
| "size": 845359182, | |
| "size_mb": 806.1973400115967, | |
| "size_gb": 0.7873020898550749, | |
| "path": "c4/c4-train.00817-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1491277837, | |
| "size_mb": 1422.193371772766, | |
| "size_gb": 1.388860714621842, | |
| "path": "common_crawl/2023-06/en_middle_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00815-of-01024.jsonl", | |
| "size": 845826510, | |
| "size_mb": 806.6430187225342, | |
| "size_gb": 0.7877373229712248, | |
| "path": "c4/c4-train.00815-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00819-of-01024.jsonl", | |
| "size": 845916630, | |
| "size_mb": 806.7289638519287, | |
| "size_gb": 0.7878212537616491, | |
| "path": "c4/c4-train.00819-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00338-of-01024.jsonl", | |
| "size": 846233314, | |
| "size_mb": 807.0309772491455, | |
| "size_gb": 0.7881161887198687, | |
| "path": "c4/c4-train.00338-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00811-of-01024.jsonl", | |
| "size": 844458044, | |
| "size_mb": 805.337947845459, | |
| "size_gb": 0.786462839692831, | |
| "path": "c4/c4-train.00811-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00269-of-01024.jsonl", | |
| "size": 846232596, | |
| "size_mb": 807.0302925109863, | |
| "size_gb": 0.7881155200302601, | |
| "path": "c4/c4-train.00269-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00014-of-01024.jsonl", | |
| "size": 847861427, | |
| "size_mb": 808.5836668014526, | |
| "size_gb": 0.7896324871107936, | |
| "path": "c4/c4-train.00014-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_94b9df70-a95f-4545-be3a-5a34f7b09fb3.jsonl", | |
| "size": 1077881206, | |
| "size_mb": 1027.9476222991943, | |
| "size_gb": 1.003855099901557, | |
| "path": "arxiv/arxiv_94b9df70-a95f-4545-be3a-5a34f7b09fb3.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00822-of-01024.jsonl", | |
| "size": 844291051, | |
| "size_mb": 805.1786909103394, | |
| "size_gb": 0.7863073153421283, | |
| "path": "c4/c4-train.00822-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1522736629, | |
| "size_mb": 1452.1948137283325, | |
| "size_gb": 1.4181589977815747, | |
| "path": "common_crawl/2023-06/en_middle_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00340-of-01024.jsonl", | |
| "size": 843865105, | |
| "size_mb": 804.7724771499634, | |
| "size_gb": 0.7859106222167611, | |
| "path": "c4/c4-train.00340-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00344-of-01024.jsonl", | |
| "size": 845671496, | |
| "size_mb": 806.4951858520508, | |
| "size_gb": 0.7875929549336433, | |
| "path": "c4/c4-train.00344-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00341-of-01024.jsonl", | |
| "size": 849214896, | |
| "size_mb": 809.8744354248047, | |
| "size_gb": 0.7908930033445358, | |
| "path": "c4/c4-train.00341-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00343-of-01024.jsonl", | |
| "size": 842733004, | |
| "size_mb": 803.6928215026855, | |
| "size_gb": 0.7848562709987164, | |
| "path": "c4/c4-train.00343-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_de744938-fa6c-45dd-b600-428dd7c63a73.jsonl", | |
| "size": 960832708, | |
| "size_mb": 916.321475982666, | |
| "size_gb": 0.8948451913893223, | |
| "path": "arxiv/arxiv_de744938-fa6c-45dd-b600-428dd7c63a73.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_de42e348-b333-4d35-b883-9bfc94f29822.jsonl", | |
| "size": 993293658, | |
| "size_mb": 947.2786502838135, | |
| "size_gb": 0.9250768069177866, | |
| "path": "arxiv/arxiv_de42e348-b333-4d35-b883-9bfc94f29822.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00336-of-01024.jsonl", | |
| "size": 846688294, | |
| "size_mb": 807.464879989624, | |
| "size_gb": 0.7885399218648672, | |
| "path": "c4/c4-train.00336-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2109875369, | |
| "size_mb": 2012.1339502334595, | |
| "size_gb": 1.9649745607748628, | |
| "path": "common_crawl/2023-06/en_head_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2215047357, | |
| "size_mb": 2112.433773994446, | |
| "size_gb": 2.062923607416451, | |
| "path": "common_crawl/2023-06/en_head_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1468447553, | |
| "size_mb": 1400.4207162857056, | |
| "size_gb": 1.3675983557477593, | |
| "path": "common_crawl/2023-06/en_middle_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1394879705, | |
| "size_mb": 1330.2609491348267, | |
| "size_gb": 1.2990829581394792, | |
| "path": "common_crawl/2019-30/en_middle_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00011-of-01024.jsonl", | |
| "size": 847272426, | |
| "size_mb": 808.021951675415, | |
| "size_gb": 0.7890839371830225, | |
| "path": "c4/c4-train.00011-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1502548369, | |
| "size_mb": 1432.9417886734009, | |
| "size_gb": 1.399357215501368, | |
| "path": "common_crawl/2023-06/en_middle_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1538943451, | |
| "size_mb": 1467.6508436203003, | |
| "size_gb": 1.4332527769729495, | |
| "path": "common_crawl/2023-06/en_middle_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_f141b736-5ce4-4f18-bb29-704227ca4bd1.jsonl", | |
| "size": 854937035, | |
| "size_mb": 815.3314924240112, | |
| "size_gb": 0.7962221605703235, | |
| "path": "arxiv/arxiv_f141b736-5ce4-4f18-bb29-704227ca4bd1.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00825-of-01024.jsonl", | |
| "size": 845605349, | |
| "size_mb": 806.4321031570435, | |
| "size_gb": 0.7875313507393003, | |
| "path": "c4/c4-train.00825-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_dbd4105f-7cbb-4483-a7b2-96b17b7fb594.jsonl", | |
| "size": 876904081, | |
| "size_mb": 836.2809000015259, | |
| "size_gb": 0.8166805664077401, | |
| "path": "arxiv/arxiv_dbd4105f-7cbb-4483-a7b2-96b17b7fb594.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00348-of-01024.jsonl", | |
| "size": 849306048, | |
| "size_mb": 809.9613647460938, | |
| "size_gb": 0.7909778952598572, | |
| "path": "c4/c4-train.00348-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00829-of-01024.jsonl", | |
| "size": 848103119, | |
| "size_mb": 808.8141622543335, | |
| "size_gb": 0.7898575803264976, | |
| "path": "c4/c4-train.00829-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_bbfabe6b-b9bc-476b-b8f0-7d6c47e9d2be.jsonl", | |
| "size": 918321702, | |
| "size_mb": 875.779821395874, | |
| "size_gb": 0.8552537318319082, | |
| "path": "arxiv/arxiv_bbfabe6b-b9bc-476b-b8f0-7d6c47e9d2be.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1475913908, | |
| "size_mb": 1407.541187286377, | |
| "size_gb": 1.3745519407093525, | |
| "path": "common_crawl/2023-06/en_middle_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00346-of-01024.jsonl", | |
| "size": 838940054, | |
| "size_mb": 800.0755825042725, | |
| "size_gb": 0.7813238110393286, | |
| "path": "c4/c4-train.00346-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00012-of-01024.jsonl", | |
| "size": 849193217, | |
| "size_mb": 809.8537607192993, | |
| "size_gb": 0.7908728132024407, | |
| "path": "c4/c4-train.00012-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00083-of-01024.jsonl", | |
| "size": 845960594, | |
| "size_mb": 806.7708911895752, | |
| "size_gb": 0.7878621984273195, | |
| "path": "c4/c4-train.00083-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1290924994, | |
| "size_mb": 1231.1220111846924, | |
| "size_gb": 1.2022675890475512, | |
| "path": "common_crawl/2020-05/en_middle_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00532-of-01024.jsonl", | |
| "size": 842695307, | |
| "size_mb": 803.65687084198, | |
| "size_gb": 0.7848211629316211, | |
| "path": "c4/c4-train.00532-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00350-of-01024.jsonl", | |
| "size": 846433212, | |
| "size_mb": 807.2216148376465, | |
| "size_gb": 0.7883023582398891, | |
| "path": "c4/c4-train.00350-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00827-of-01024.jsonl", | |
| "size": 846513732, | |
| "size_mb": 807.2984046936035, | |
| "size_gb": 0.7883773483335972, | |
| "path": "c4/c4-train.00827-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00111-of-01024.jsonl", | |
| "size": 845096341, | |
| "size_mb": 805.9466753005981, | |
| "size_gb": 0.7870573000982404, | |
| "path": "c4/c4-train.00111-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00059-of-01024.jsonl", | |
| "size": 843077297, | |
| "size_mb": 804.021164894104, | |
| "size_gb": 0.7851769188418984, | |
| "path": "c4/c4-train.00059-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00534-of-01024.jsonl", | |
| "size": 844746777, | |
| "size_mb": 805.6133050918579, | |
| "size_gb": 0.7867317432537675, | |
| "path": "c4/c4-train.00534-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00099-of-01024.jsonl", | |
| "size": 847608090, | |
| "size_mb": 808.3420658111572, | |
| "size_gb": 0.7893965486437082, | |
| "path": "c4/c4-train.00099-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00032-of-01024.jsonl", | |
| "size": 849567999, | |
| "size_mb": 810.2111806869507, | |
| "size_gb": 0.7912218561396003, | |
| "path": "c4/c4-train.00032-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00536-of-01024.jsonl", | |
| "size": 847106832, | |
| "size_mb": 807.8640289306641, | |
| "size_gb": 0.7889297157526016, | |
| "path": "c4/c4-train.00536-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00048-of-01024.jsonl", | |
| "size": 841827750, | |
| "size_mb": 802.8295040130615, | |
| "size_gb": 0.7840131875127554, | |
| "path": "c4/c4-train.00048-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00115-of-01024.jsonl", | |
| "size": 844474811, | |
| "size_mb": 805.3539381027222, | |
| "size_gb": 0.7864784551784396, | |
| "path": "c4/c4-train.00115-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0081.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1579589948, | |
| "size_mb": 1506.414363861084, | |
| "size_gb": 1.4711077772080898, | |
| "path": "common_crawl/2021-04/en_middle_0081.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1541412595, | |
| "size_mb": 1470.005602836609, | |
| "size_gb": 1.4355523465201259, | |
| "path": "common_crawl/2023-06/en_middle_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1501907345, | |
| "size_mb": 1432.3304605484009, | |
| "size_gb": 1.3987602153792977, | |
| "path": "common_crawl/2023-06/en_middle_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1501895218, | |
| "size_mb": 1432.3188953399658, | |
| "size_gb": 1.3987489212304354, | |
| "path": "common_crawl/2023-06/en_middle_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00531-of-01024.jsonl", | |
| "size": 846775296, | |
| "size_mb": 807.5478515625, | |
| "size_gb": 0.7886209487915039, | |
| "path": "c4/c4-train.00531-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00064-of-01024.jsonl", | |
| "size": 845680861, | |
| "size_mb": 806.5041170120239, | |
| "size_gb": 0.7876016767695546, | |
| "path": "c4/c4-train.00064-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00037-of-01024.jsonl", | |
| "size": 847677439, | |
| "size_mb": 808.4082021713257, | |
| "size_gb": 0.7894611349329352, | |
| "path": "c4/c4-train.00037-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00538-of-01024.jsonl", | |
| "size": 848282000, | |
| "size_mb": 808.9847564697266, | |
| "size_gb": 0.7900241762399673, | |
| "path": "c4/c4-train.00538-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00105-of-01024.jsonl", | |
| "size": 849847481, | |
| "size_mb": 810.4777154922485, | |
| "size_gb": 0.791482144035399, | |
| "path": "c4/c4-train.00105-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00049-of-01024.jsonl", | |
| "size": 846461937, | |
| "size_mb": 807.2490091323853, | |
| "size_gb": 0.788329110480845, | |
| "path": "c4/c4-train.00049-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00112-of-01024.jsonl", | |
| "size": 845270931, | |
| "size_mb": 806.1131772994995, | |
| "size_gb": 0.7872198997065425, | |
| "path": "c4/c4-train.00112-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00056-of-01024.jsonl", | |
| "size": 847208466, | |
| "size_mb": 807.9609546661377, | |
| "size_gb": 0.7890243697911501, | |
| "path": "c4/c4-train.00056-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00085-of-01024.jsonl", | |
| "size": 849741596, | |
| "size_mb": 810.3767356872559, | |
| "size_gb": 0.7913835309445858, | |
| "path": "c4/c4-train.00085-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00537-of-01024.jsonl", | |
| "size": 846056652, | |
| "size_mb": 806.8624992370605, | |
| "size_gb": 0.7879516594111919, | |
| "path": "c4/c4-train.00537-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1522290250, | |
| "size_mb": 1451.7691135406494, | |
| "size_gb": 1.4177432749420404, | |
| "path": "common_crawl/2023-06/en_middle_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2192969807, | |
| "size_mb": 2091.378981590271, | |
| "size_gb": 2.042362286709249, | |
| "path": "common_crawl/2023-06/en_head_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1537278691, | |
| "size_mb": 1466.0632047653198, | |
| "size_gb": 1.4317023484036326, | |
| "path": "common_crawl/2023-06/en_middle_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00061-of-01024.jsonl", | |
| "size": 845367722, | |
| "size_mb": 806.2054843902588, | |
| "size_gb": 0.7873100433498621, | |
| "path": "c4/c4-train.00061-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0071.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1533906661, | |
| "size_mb": 1462.8473863601685, | |
| "size_gb": 1.428561900742352, | |
| "path": "common_crawl/2023-06/en_middle_0071.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2019660444, | |
| "size_mb": 1926.0982933044434, | |
| "size_gb": 1.8809553645551205, | |
| "path": "common_crawl/2022-05/en_head_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00533-of-01024.jsonl", | |
| "size": 844910039, | |
| "size_mb": 805.769003868103, | |
| "size_gb": 0.7868837928399444, | |
| "path": "c4/c4-train.00533-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00019-of-01024.jsonl", | |
| "size": 847239690, | |
| "size_mb": 807.9907321929932, | |
| "size_gb": 0.7890534494072199, | |
| "path": "c4/c4-train.00019-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00535-of-01024.jsonl", | |
| "size": 846004302, | |
| "size_mb": 806.8125743865967, | |
| "size_gb": 0.7879029046744108, | |
| "path": "c4/c4-train.00535-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1528606634, | |
| "size_mb": 1457.7928867340088, | |
| "size_gb": 1.4236258659511805, | |
| "path": "common_crawl/2023-06/en_middle_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00118-of-01024.jsonl", | |
| "size": 845489046, | |
| "size_mb": 806.3211879730225, | |
| "size_gb": 0.7874230351299047, | |
| "path": "c4/c4-train.00118-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00828-of-01024.jsonl", | |
| "size": 847244454, | |
| "size_mb": 807.9952754974365, | |
| "size_gb": 0.7890578862279654, | |
| "path": "c4/c4-train.00828-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0127.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1290369187, | |
| "size_mb": 1230.5919523239136, | |
| "size_gb": 1.2017499534413218, | |
| "path": "common_crawl/2020-05/en_middle_0127.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00123-of-01024.jsonl", | |
| "size": 843478320, | |
| "size_mb": 804.4036102294922, | |
| "size_gb": 0.7855504006147385, | |
| "path": "c4/c4-train.00123-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00349-of-01024.jsonl", | |
| "size": 847821882, | |
| "size_mb": 808.5459537506104, | |
| "size_gb": 0.7895956579595804, | |
| "path": "c4/c4-train.00349-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00345-of-01024.jsonl", | |
| "size": 844087799, | |
| "size_mb": 804.9848546981812, | |
| "size_gb": 0.7861180221661925, | |
| "path": "c4/c4-train.00345-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00824-of-01024.jsonl", | |
| "size": 845589310, | |
| "size_mb": 806.4168071746826, | |
| "size_gb": 0.787516413256526, | |
| "path": "c4/c4-train.00824-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1499246834, | |
| "size_mb": 1429.7931995391846, | |
| "size_gb": 1.396282421424985, | |
| "path": "common_crawl/2023-06/en_middle_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1464218370, | |
| "size_mb": 1396.3874530792236, | |
| "size_gb": 1.3636596221476793, | |
| "path": "common_crawl/2023-06/en_middle_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1539120485, | |
| "size_mb": 1467.819676399231, | |
| "size_gb": 1.433417652733624, | |
| "path": "common_crawl/2023-06/en_middle_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0128.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 283167009, | |
| "size_mb": 270.04910373687744, | |
| "size_gb": 0.2637198278680444, | |
| "path": "common_crawl/2020-05/en_middle_0128.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0126.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1270332381, | |
| "size_mb": 1211.483365058899, | |
| "size_gb": 1.183089223690331, | |
| "path": "common_crawl/2020-05/en_middle_0126.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1470686244, | |
| "size_mb": 1402.5556983947754, | |
| "size_gb": 1.3696832992136478, | |
| "path": "common_crawl/2023-06/en_middle_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1910452816, | |
| "size_mb": 1821.9497833251953, | |
| "size_gb": 1.779247835278511, | |
| "path": "common_crawl/2021-04/en_head_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1514986920, | |
| "size_mb": 1444.8041152954102, | |
| "size_gb": 1.410941518843174, | |
| "path": "common_crawl/2023-06/en_middle_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_db3b4be7-4e98-4fe9-96bf-05a5788815e3.jsonl", | |
| "size": 870237779, | |
| "size_mb": 829.9234189987183, | |
| "size_gb": 0.8104720888659358, | |
| "path": "arxiv/arxiv_db3b4be7-4e98-4fe9-96bf-05a5788815e3.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1584909086, | |
| "size_mb": 1511.4870891571045, | |
| "size_gb": 1.4760616105049849, | |
| "path": "common_crawl/2021-04/en_middle_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0125.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1288419171, | |
| "size_mb": 1228.7322721481323, | |
| "size_gb": 1.1999338595196605, | |
| "path": "common_crawl/2020-05/en_middle_0125.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_a055bd62-1ec2-47cf-bad2-321e3d4f053f.jsonl", | |
| "size": 914464024, | |
| "size_mb": 872.1008529663086, | |
| "size_gb": 0.8516609892249107, | |
| "path": "arxiv/arxiv_a055bd62-1ec2-47cf-bad2-321e3d4f053f.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00833-of-01024.jsonl", | |
| "size": 846308663, | |
| "size_mb": 807.1028356552124, | |
| "size_gb": 0.7881863629445434, | |
| "path": "c4/c4-train.00833-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00013-of-01024.jsonl", | |
| "size": 847512208, | |
| "size_mb": 808.2506256103516, | |
| "size_gb": 0.789307251572609, | |
| "path": "c4/c4-train.00013-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00331-of-01024.jsonl", | |
| "size": 846404584, | |
| "size_mb": 807.1943130493164, | |
| "size_gb": 0.788275696337223, | |
| "path": "c4/c4-train.00331-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00823-of-01024.jsonl", | |
| "size": 844367644, | |
| "size_mb": 805.2517356872559, | |
| "size_gb": 0.7863786481320858, | |
| "path": "c4/c4-train.00823-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00342-of-01024.jsonl", | |
| "size": 847320302, | |
| "size_mb": 808.0676097869873, | |
| "size_gb": 0.7891285251826048, | |
| "path": "c4/c4-train.00342-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_ca6e3842-6ca4-4230-84fa-376a3374c380.jsonl", | |
| "size": 926049410, | |
| "size_mb": 883.1495380401611, | |
| "size_gb": 0.8624507207423449, | |
| "path": "arxiv/arxiv_ca6e3842-6ca4-4230-84fa-376a3374c380.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_95ffc9e1-c505-4a3b-8fb0-cbc98b8703e1.jsonl", | |
| "size": 831261471, | |
| "size_mb": 792.7527151107788, | |
| "size_gb": 0.7741725733503699, | |
| "path": "arxiv/arxiv_95ffc9e1-c505-4a3b-8fb0-cbc98b8703e1.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00335-of-01024.jsonl", | |
| "size": 844110267, | |
| "size_mb": 805.0062818527222, | |
| "size_gb": 0.786138947121799, | |
| "path": "c4/c4-train.00335-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00826-of-01024.jsonl", | |
| "size": 843811123, | |
| "size_mb": 804.7209959030151, | |
| "size_gb": 0.7858603475615382, | |
| "path": "c4/c4-train.00826-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_b1f3b912-a2ab-43bd-8811-43d84b422506.jsonl", | |
| "size": 1101326806, | |
| "size_mb": 1050.3070888519287, | |
| "size_gb": 1.0256905164569616, | |
| "path": "arxiv/arxiv_b1f3b912-a2ab-43bd-8811-43d84b422506.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00810-of-01024.jsonl", | |
| "size": 847119122, | |
| "size_mb": 807.8757495880127, | |
| "size_gb": 0.7889411617070436, | |
| "path": "c4/c4-train.00810-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00832-of-01024.jsonl", | |
| "size": 844644187, | |
| "size_mb": 805.5154676437378, | |
| "size_gb": 0.7866361988708377, | |
| "path": "c4/c4-train.00832-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00294-of-01024.jsonl", | |
| "size": 845517804, | |
| "size_mb": 806.3486137390137, | |
| "size_gb": 0.7874498181045055, | |
| "path": "c4/c4-train.00294-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_c44164b4-0770-48d0-87db-590ca529032a.jsonl", | |
| "size": 892841729, | |
| "size_mb": 851.4802255630493, | |
| "size_gb": 0.8315236577764153, | |
| "path": "arxiv/arxiv_c44164b4-0770-48d0-87db-590ca529032a.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00818-of-01024.jsonl", | |
| "size": 847585470, | |
| "size_mb": 808.3204936981201, | |
| "size_gb": 0.7893754821270704, | |
| "path": "c4/c4-train.00818-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00337-of-01024.jsonl", | |
| "size": 848846815, | |
| "size_mb": 809.5234060287476, | |
| "size_gb": 0.7905502011999488, | |
| "path": "c4/c4-train.00337-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00361-of-01024.jsonl", | |
| "size": 844922440, | |
| "size_mb": 805.7808303833008, | |
| "size_gb": 0.7868953421711922, | |
| "path": "c4/c4-train.00361-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00015-of-01024.jsonl", | |
| "size": 844100876, | |
| "size_mb": 804.9973258972168, | |
| "size_gb": 0.7861302010715008, | |
| "path": "c4/c4-train.00015-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01011-of-01024.jsonl", | |
| "size": 848802965, | |
| "size_mb": 809.4815874099731, | |
| "size_gb": 0.7905093627050519, | |
| "path": "c4/c4-train.01011-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1519611492, | |
| "size_mb": 1449.2144508361816, | |
| "size_gb": 1.4152484871447086, | |
| "path": "common_crawl/2023-06/en_middle_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00353-of-01024.jsonl", | |
| "size": 850420778, | |
| "size_mb": 811.0244541168213, | |
| "size_gb": 0.7920160684734583, | |
| "path": "c4/c4-train.00353-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00002-of-01024.jsonl", | |
| "size": 847693903, | |
| "size_mb": 808.423903465271, | |
| "size_gb": 0.7894764682278037, | |
| "path": "c4/c4-train.00002-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00814-of-01024.jsonl", | |
| "size": 846960975, | |
| "size_mb": 807.724928855896, | |
| "size_gb": 0.7887938758358359, | |
| "path": "c4/c4-train.00814-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_a8fcae75b0c3410faabcff02f0056a36.sampled.jsonl", | |
| "size": 2724207104, | |
| "size_mb": 2598.00634765625, | |
| "size_gb": 2.5371155738830566, | |
| "path": "github/filtered_a8fcae75b0c3410faabcff02f0056a36.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00821-of-01024.jsonl", | |
| "size": 851629791, | |
| "size_mb": 812.1774587631226, | |
| "size_gb": 0.7931420495733619, | |
| "path": "c4/c4-train.00821-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00339-of-01024.jsonl", | |
| "size": 847590725, | |
| "size_mb": 808.3255052566528, | |
| "size_gb": 0.7893803762272, | |
| "path": "c4/c4-train.00339-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00816-of-01024.jsonl", | |
| "size": 849044985, | |
| "size_mb": 809.7123956680298, | |
| "size_gb": 0.7907347613945603, | |
| "path": "c4/c4-train.00816-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1517546613, | |
| "size_mb": 1447.245228767395, | |
| "size_gb": 1.4133254187181592, | |
| "path": "common_crawl/2023-06/en_middle_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00841-of-01024.jsonl", | |
| "size": 845188123, | |
| "size_mb": 806.0342054367065, | |
| "size_gb": 0.7871427787467837, | |
| "path": "c4/c4-train.00841-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_e8f25867-697d-4f52-84e1-e50a95bc182b.jsonl", | |
| "size": 1053552716, | |
| "size_mb": 1004.746166229248, | |
| "size_gb": 0.98119742795825, | |
| "path": "arxiv/arxiv_e8f25867-697d-4f52-84e1-e50a95bc182b.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00352-of-01024.jsonl", | |
| "size": 845058094, | |
| "size_mb": 805.9102001190186, | |
| "size_gb": 0.7870216798037291, | |
| "path": "c4/c4-train.00352-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2109852861, | |
| "size_mb": 2012.1124849319458, | |
| "size_gb": 1.9649535985663533, | |
| "path": "common_crawl/2021-04/en_head_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1568270099, | |
| "size_mb": 1495.618914604187, | |
| "size_gb": 1.4605653462931514, | |
| "path": "common_crawl/2023-06/en_middle_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00831-of-01024.jsonl", | |
| "size": 845887603, | |
| "size_mb": 806.7012815475464, | |
| "size_gb": 0.7877942202612758, | |
| "path": "c4/c4-train.00831-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_d0da78e9-3dcf-4a46-85c1-f23ed00178bc.jsonl", | |
| "size": 985513715, | |
| "size_mb": 939.8591184616089, | |
| "size_gb": 0.9178311703726649, | |
| "path": "arxiv/arxiv_d0da78e9-3dcf-4a46-85c1-f23ed00178bc.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1514609476, | |
| "size_mb": 1444.4441566467285, | |
| "size_gb": 1.4105899967253208, | |
| "path": "common_crawl/2023-06/en_middle_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00820-of-01024.jsonl", | |
| "size": 848240692, | |
| "size_mb": 808.9453620910645, | |
| "size_gb": 0.7899857051670551, | |
| "path": "c4/c4-train.00820-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1904434337, | |
| "size_mb": 1816.210114479065, | |
| "size_gb": 1.7736426899209619, | |
| "path": "common_crawl/2021-04/en_head_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/stackexchange/stackexchange.jsonl", | |
| "size": 80046059672, | |
| "size_mb": 76337.8712387085, | |
| "size_gb": 74.54870238155127, | |
| "path": "stackexchange/stackexchange.jsonl", | |
| "top_folder": "stackexchange", | |
| "top_folders": "stackexchange" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01015-of-01024.jsonl", | |
| "size": 846210235, | |
| "size_mb": 807.0089673995972, | |
| "size_gb": 0.7880946947261691, | |
| "path": "c4/c4-train.01015-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1512928761, | |
| "size_mb": 1442.8413019180298, | |
| "size_gb": 1.409024708904326, | |
| "path": "common_crawl/2023-06/en_middle_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_90fa9c2b-25b0-47b7-af2b-a683356e543b.jsonl", | |
| "size": 940149219, | |
| "size_mb": 896.5961637496948, | |
| "size_gb": 0.8755821911618114, | |
| "path": "arxiv/arxiv_90fa9c2b-25b0-47b7-af2b-a683356e543b.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00780-of-01024.jsonl", | |
| "size": 841544257, | |
| "size_mb": 802.5591440200806, | |
| "size_gb": 0.7837491640821099, | |
| "path": "c4/c4-train.00780-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01020-of-01024.jsonl", | |
| "size": 841411853, | |
| "size_mb": 802.4328737258911, | |
| "size_gb": 0.7836258532479405, | |
| "path": "c4/c4-train.01020-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00301-of-01024.jsonl", | |
| "size": 846090756, | |
| "size_mb": 806.8950233459473, | |
| "size_gb": 0.7879834212362766, | |
| "path": "c4/c4-train.00301-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1537130108, | |
| "size_mb": 1465.9215049743652, | |
| "size_gb": 1.4315639697015285, | |
| "path": "common_crawl/2023-06/en_middle_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1845484873, | |
| "size_mb": 1759.99152469635, | |
| "size_gb": 1.7187417233362794, | |
| "path": "common_crawl/2021-04/en_head_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00299-of-01024.jsonl", | |
| "size": 847726277, | |
| "size_mb": 808.4547777175903, | |
| "size_gb": 0.7895066188648343, | |
| "path": "c4/c4-train.00299-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00300-of-01024.jsonl", | |
| "size": 847968484, | |
| "size_mb": 808.6857643127441, | |
| "size_gb": 0.7897321917116642, | |
| "path": "c4/c4-train.00300-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_fbfc552e48164acda6605fa31fc2f563.sampled.jsonl", | |
| "size": 1748415821, | |
| "size_mb": 1667.4192628860474, | |
| "size_gb": 1.6283391239121556, | |
| "path": "github/filtered_fbfc552e48164acda6605fa31fc2f563.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0074.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1498184451, | |
| "size_mb": 1428.780032157898, | |
| "size_gb": 1.3952930001541972, | |
| "path": "common_crawl/2023-06/en_middle_0074.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01021-of-01024.jsonl", | |
| "size": 843699515, | |
| "size_mb": 804.6145582199097, | |
| "size_gb": 0.7857564045116305, | |
| "path": "c4/c4-train.01021-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_middle_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1457585345, | |
| "size_mb": 1390.061707496643, | |
| "size_gb": 1.3574821362271905, | |
| "path": "common_crawl/2022-05/en_middle_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1513936430, | |
| "size_mb": 1443.8022899627686, | |
| "size_gb": 1.4099631737917662, | |
| "path": "common_crawl/2023-06/en_middle_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_024de5df-1b7f-447c-8c3a-51407d8d6732.jsonl", | |
| "size": 829033523, | |
| "size_mb": 790.6279783248901, | |
| "size_gb": 0.7720976350829005, | |
| "path": "arxiv/arxiv_024de5df-1b7f-447c-8c3a-51407d8d6732.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0073.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1518644389, | |
| "size_mb": 1448.2921495437622, | |
| "size_gb": 1.4143478022888303, | |
| "path": "common_crawl/2023-06/en_middle_0073.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_fc817e4c16494957bcb89b166e91434f.sampled.jsonl", | |
| "size": 2021817493, | |
| "size_mb": 1928.1554155349731, | |
| "size_gb": 1.8829642729833722, | |
| "path": "github/filtered_fc817e4c16494957bcb89b166e91434f.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00899-of-01024.jsonl", | |
| "size": 847886072, | |
| "size_mb": 808.6071701049805, | |
| "size_gb": 0.789655439555645, | |
| "path": "c4/c4-train.00899-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1527961628, | |
| "size_mb": 1457.1777610778809, | |
| "size_gb": 1.423025157302618, | |
| "path": "common_crawl/2023-06/en_middle_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1388776709, | |
| "size_mb": 1324.4406785964966, | |
| "size_gb": 1.2933991001918912, | |
| "path": "common_crawl/2019-30/en_middle_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1405306676, | |
| "size_mb": 1340.2048835754395, | |
| "size_gb": 1.30879383161664, | |
| "path": "common_crawl/2019-30/en_middle_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1481764149, | |
| "size_mb": 1413.1204118728638, | |
| "size_gb": 1.3800004022195935, | |
| "path": "common_crawl/2023-06/en_middle_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_6719ecd2-fe34-4078-a584-320d921cbf6f.jsonl", | |
| "size": 894498965, | |
| "size_mb": 853.0606889724731, | |
| "size_gb": 0.8330670790746808, | |
| "path": "arxiv/arxiv_6719ecd2-fe34-4078-a584-320d921cbf6f.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00278-of-01024.jsonl", | |
| "size": 847872771, | |
| "size_mb": 808.594485282898, | |
| "size_gb": 0.78964305203408, | |
| "path": "c4/c4-train.00278-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1341562330, | |
| "size_mb": 1279.413537979126, | |
| "size_gb": 1.2494272831827402, | |
| "path": "common_crawl/2020-05/en_middle_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00999-of-01024.jsonl", | |
| "size": 845451192, | |
| "size_mb": 806.2850875854492, | |
| "size_gb": 0.7873877808451653, | |
| "path": "c4/c4-train.00999-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00279-of-01024.jsonl", | |
| "size": 846606899, | |
| "size_mb": 807.3872556686401, | |
| "size_gb": 0.7884641168639064, | |
| "path": "c4/c4-train.00279-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1571751534, | |
| "size_mb": 1498.9390697479248, | |
| "size_gb": 1.4638076853007078, | |
| "path": "common_crawl/2023-06/en_middle_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00759-of-01024.jsonl", | |
| "size": 843078589, | |
| "size_mb": 804.0223970413208, | |
| "size_gb": 0.7851781221106648, | |
| "path": "c4/c4-train.00759-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00758-of-01024.jsonl", | |
| "size": 846305990, | |
| "size_mb": 807.1002864837646, | |
| "size_gb": 0.7881838735193014, | |
| "path": "c4/c4-train.00758-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1473766697, | |
| "size_mb": 1405.493447303772, | |
| "size_gb": 1.3725521946325898, | |
| "path": "common_crawl/2023-06/en_middle_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01003-of-01024.jsonl", | |
| "size": 843880422, | |
| "size_mb": 804.7870845794678, | |
| "size_gb": 0.7859248872846365, | |
| "path": "c4/c4-train.01003-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1539882360, | |
| "size_mb": 1468.546257019043, | |
| "size_gb": 1.4341272041201591, | |
| "path": "common_crawl/2023-06/en_middle_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00874-of-01024.jsonl", | |
| "size": 845260513, | |
| "size_mb": 806.1032419204712, | |
| "size_gb": 0.7872101971879601, | |
| "path": "c4/c4-train.00874-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_6938ee72-43ee-4ade-8840-151a402383b0.jsonl", | |
| "size": 862418985, | |
| "size_mb": 822.466835975647, | |
| "size_gb": 0.8031902695074677, | |
| "path": "arxiv/arxiv_6938ee72-43ee-4ade-8840-151a402383b0.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00885-of-01024.jsonl", | |
| "size": 845863826, | |
| "size_mb": 806.6786060333252, | |
| "size_gb": 0.7877720762044191, | |
| "path": "c4/c4-train.00885-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_1d750c0ce39d40c6bc20bad9469e5a99.sampled.jsonl", | |
| "size": 2487418424, | |
| "size_mb": 2372.1870651245117, | |
| "size_gb": 2.316588930785656, | |
| "path": "github/filtered_1d750c0ce39d40c6bc20bad9469e5a99.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00489-of-01024.jsonl", | |
| "size": 846511219, | |
| "size_mb": 807.2960081100464, | |
| "size_gb": 0.7883750079199672, | |
| "path": "c4/c4-train.00489-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00490-of-01024.jsonl", | |
| "size": 847799101, | |
| "size_mb": 808.5242280960083, | |
| "size_gb": 0.7895744415000081, | |
| "path": "c4/c4-train.00490-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00730-of-01024.jsonl", | |
| "size": 843456609, | |
| "size_mb": 804.3829050064087, | |
| "size_gb": 0.785530180670321, | |
| "path": "c4/c4-train.00730-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00729-of-01024.jsonl", | |
| "size": 848869196, | |
| "size_mb": 809.544750213623, | |
| "size_gb": 0.7905710451304913, | |
| "path": "c4/c4-train.00729-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00354-of-01024.jsonl", | |
| "size": 847340242, | |
| "size_mb": 808.0866260528564, | |
| "size_gb": 0.7891470957547426, | |
| "path": "c4/c4-train.00354-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00129-of-01024.jsonl", | |
| "size": 844076560, | |
| "size_mb": 804.9741363525391, | |
| "size_gb": 0.7861075550317764, | |
| "path": "c4/c4-train.00129-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1403082313, | |
| "size_mb": 1338.083565711975, | |
| "size_gb": 1.3067222321406007, | |
| "path": "common_crawl/2019-30/en_middle_0060.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_a94ea420-99ae-4d58-9cdc-d4666e3322a7.jsonl", | |
| "size": 969340435, | |
| "size_mb": 924.435076713562, | |
| "size_gb": 0.9027686296030879, | |
| "path": "arxiv/arxiv_a94ea420-99ae-4d58-9cdc-d4666e3322a7.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00884-of-01024.jsonl", | |
| "size": 846482051, | |
| "size_mb": 807.2681913375854, | |
| "size_gb": 0.7883478431031108, | |
| "path": "c4/c4-train.00884-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00891-of-01024.jsonl", | |
| "size": 842447839, | |
| "size_mb": 803.4208669662476, | |
| "size_gb": 0.7845906903967261, | |
| "path": "c4/c4-train.00891-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00887-of-01024.jsonl", | |
| "size": 844798160, | |
| "size_mb": 805.6623077392578, | |
| "size_gb": 0.786779597401619, | |
| "path": "c4/c4-train.00887-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00990-of-01024.jsonl", | |
| "size": 846507910, | |
| "size_mb": 807.2928524017334, | |
| "size_gb": 0.7883719261735678, | |
| "path": "c4/c4-train.00990-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00270-of-01024.jsonl", | |
| "size": 846448735, | |
| "size_mb": 807.2364187240601, | |
| "size_gb": 0.7883168151602149, | |
| "path": "c4/c4-train.00270-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00888-of-01024.jsonl", | |
| "size": 847663347, | |
| "size_mb": 808.3947629928589, | |
| "size_gb": 0.7894480107352138, | |
| "path": "c4/c4-train.00888-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00347-of-01024.jsonl", | |
| "size": 845445402, | |
| "size_mb": 806.2795658111572, | |
| "size_gb": 0.7873823884874582, | |
| "path": "c4/c4-train.00347-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00950-of-01024.jsonl", | |
| "size": 847646085, | |
| "size_mb": 808.3783006668091, | |
| "size_gb": 0.7894319342449307, | |
| "path": "c4/c4-train.00950-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1363504878, | |
| "size_mb": 1300.3395824432373, | |
| "size_gb": 1.269862873479724, | |
| "path": "common_crawl/2020-05/en_middle_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00883-of-01024.jsonl", | |
| "size": 849259171, | |
| "size_mb": 809.9166593551636, | |
| "size_gb": 0.7909342376515269, | |
| "path": "c4/c4-train.00883-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_751370b5-c7cb-44d8-a039-1468ee6747ab.jsonl", | |
| "size": 1027326012, | |
| "size_mb": 979.734432220459, | |
| "size_gb": 0.956771906465292, | |
| "path": "arxiv/arxiv_751370b5-c7cb-44d8-a039-1468ee6747ab.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00750-of-01024.jsonl", | |
| "size": 846484228, | |
| "size_mb": 807.2702674865723, | |
| "size_gb": 0.7883498705923557, | |
| "path": "c4/c4-train.00750-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00742-of-01024.jsonl", | |
| "size": 847175559, | |
| "size_mb": 807.9295721054077, | |
| "size_gb": 0.7889937227591872, | |
| "path": "c4/c4-train.00742-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00744-of-01024.jsonl", | |
| "size": 849748957, | |
| "size_mb": 810.3837556838989, | |
| "size_gb": 0.7913903864100575, | |
| "path": "c4/c4-train.00744-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00283-of-01024.jsonl", | |
| "size": 846405314, | |
| "size_mb": 807.1950092315674, | |
| "size_gb": 0.7882763762027025, | |
| "path": "c4/c4-train.00283-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_b821f640b8f14ed588bf48ae13f44098.sampled.jsonl", | |
| "size": 2040033224, | |
| "size_mb": 1945.5272903442383, | |
| "size_gb": 1.8999289944767952, | |
| "path": "github/filtered_b821f640b8f14ed588bf48ae13f44098.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00886-of-01024.jsonl", | |
| "size": 844583831, | |
| "size_mb": 805.4579076766968, | |
| "size_gb": 0.7865799879655242, | |
| "path": "c4/c4-train.00886-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00504-of-01024.jsonl", | |
| "size": 847169874, | |
| "size_mb": 807.924150466919, | |
| "size_gb": 0.7889884281903505, | |
| "path": "c4/c4-train.00504-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_d386838c-a51e-4839-9f27-8495b2466e49.jsonl", | |
| "size": 975046805, | |
| "size_mb": 929.8770952224731, | |
| "size_gb": 0.9080831008031964, | |
| "path": "arxiv/arxiv_d386838c-a51e-4839-9f27-8495b2466e49.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1493078740, | |
| "size_mb": 1423.910846710205, | |
| "size_gb": 1.3905379362404346, | |
| "path": "common_crawl/2023-06/en_middle_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2222799244, | |
| "size_mb": 2119.8265495300293, | |
| "size_gb": 2.0701431147754192, | |
| "path": "common_crawl/2023-06/en_head_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0083.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1406393782, | |
| "size_mb": 1341.2416286468506, | |
| "size_gb": 1.30980627797544, | |
| "path": "common_crawl/2020-05/en_middle_0083.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00937-of-01024.jsonl", | |
| "size": 843393750, | |
| "size_mb": 804.3229579925537, | |
| "size_gb": 0.7854716386646032, | |
| "path": "c4/c4-train.00937-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00939-of-01024.jsonl", | |
| "size": 850314652, | |
| "size_mb": 810.9232444763184, | |
| "size_gb": 0.7919172309339046, | |
| "path": "c4/c4-train.00939-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00998-of-01024.jsonl", | |
| "size": 847729699, | |
| "size_mb": 808.4580411911011, | |
| "size_gb": 0.7895098058506846, | |
| "path": "c4/c4-train.00998-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00881-of-01024.jsonl", | |
| "size": 847706848, | |
| "size_mb": 808.4362487792969, | |
| "size_gb": 0.7894885241985321, | |
| "path": "c4/c4-train.00881-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2158535880, | |
| "size_mb": 2058.5402297973633, | |
| "size_gb": 2.0102931931614876, | |
| "path": "common_crawl/2022-05/en_head_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2203511146, | |
| "size_mb": 2101.4319858551025, | |
| "size_gb": 2.0521796736866236, | |
| "path": "common_crawl/2023-06/en_head_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00936-of-01024.jsonl", | |
| "size": 843819410, | |
| "size_mb": 804.7288990020752, | |
| "size_gb": 0.7858680654317141, | |
| "path": "c4/c4-train.00936-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00879-of-01024.jsonl", | |
| "size": 844924856, | |
| "size_mb": 805.7831344604492, | |
| "size_gb": 0.7868975922465324, | |
| "path": "c4/c4-train.00879-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0072.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1569026228, | |
| "size_mb": 1496.340015411377, | |
| "size_gb": 1.4612695463001728, | |
| "path": "common_crawl/2023-06/en_middle_0072.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00880-of-01024.jsonl", | |
| "size": 844873941, | |
| "size_mb": 805.7345781326294, | |
| "size_gb": 0.7868501739576459, | |
| "path": "c4/c4-train.00880-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00942-of-01024.jsonl", | |
| "size": 848477722, | |
| "size_mb": 809.1714115142822, | |
| "size_gb": 0.7902064565569162, | |
| "path": "c4/c4-train.00942-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00878-of-01024.jsonl", | |
| "size": 845380069, | |
| "size_mb": 806.2172594070435, | |
| "size_gb": 0.7873215423896909, | |
| "path": "c4/c4-train.00878-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00941-of-01024.jsonl", | |
| "size": 846401715, | |
| "size_mb": 807.1915769577026, | |
| "size_gb": 0.7882730243727565, | |
| "path": "c4/c4-train.00941-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00877-of-01024.jsonl", | |
| "size": 848932260, | |
| "size_mb": 809.6048927307129, | |
| "size_gb": 0.7906297780573368, | |
| "path": "c4/c4-train.00877-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00938-of-01024.jsonl", | |
| "size": 846021750, | |
| "size_mb": 806.8292140960693, | |
| "size_gb": 0.7879191543906927, | |
| "path": "c4/c4-train.00938-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00882-of-01024.jsonl", | |
| "size": 841743931, | |
| "size_mb": 802.7495679855347, | |
| "size_gb": 0.7839351249858737, | |
| "path": "c4/c4-train.00882-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1492630329, | |
| "size_mb": 1423.483208656311, | |
| "size_gb": 1.3901203209534287, | |
| "path": "common_crawl/2023-06/en_middle_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1276872169, | |
| "size_mb": 1217.7201929092407, | |
| "size_gb": 1.1891798758879304, | |
| "path": "common_crawl/2020-05/en_middle_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00395-of-01024.jsonl", | |
| "size": 843908228, | |
| "size_mb": 804.8136024475098, | |
| "size_gb": 0.7859507836401463, | |
| "path": "c4/c4-train.00395-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00935-of-01024.jsonl", | |
| "size": 844940820, | |
| "size_mb": 805.7983589172363, | |
| "size_gb": 0.7869124598801136, | |
| "path": "c4/c4-train.00935-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00940-of-01024.jsonl", | |
| "size": 849678897, | |
| "size_mb": 810.3169412612915, | |
| "size_gb": 0.79132513795048, | |
| "path": "c4/c4-train.00940-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00876-of-01024.jsonl", | |
| "size": 850473185, | |
| "size_mb": 811.0744333267212, | |
| "size_gb": 0.7920648762956262, | |
| "path": "c4/c4-train.00876-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00875-of-01024.jsonl", | |
| "size": 845392610, | |
| "size_mb": 806.2292194366455, | |
| "size_gb": 0.7873332221060991, | |
| "path": "c4/c4-train.00875-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2218806484, | |
| "size_mb": 2116.018756866455, | |
| "size_gb": 2.0664245672523975, | |
| "path": "common_crawl/2022-05/en_head_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2184951326, | |
| "size_mb": 2083.7319622039795, | |
| "size_gb": 2.0348944943398237, | |
| "path": "common_crawl/2023-06/en_head_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1364167705, | |
| "size_mb": 1300.971703529358, | |
| "size_gb": 1.2704801792278886, | |
| "path": "common_crawl/2019-30/en_middle_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_head_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 972656748, | |
| "size_mb": 927.5977592468262, | |
| "size_gb": 0.9058571867644787, | |
| "path": "common_crawl/2023-06/en_head_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1272850038, | |
| "size_mb": 1213.8843898773193, | |
| "size_gb": 1.1854339744895697, | |
| "path": "common_crawl/2020-05/en_middle_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0070.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1501422245, | |
| "size_mb": 1431.8678331375122, | |
| "size_gb": 1.3983084307983518, | |
| "path": "common_crawl/2023-06/en_middle_0070.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2052420499, | |
| "size_mb": 1957.3407163619995, | |
| "size_gb": 1.9114655433222651, | |
| "path": "common_crawl/2020-05/en_head_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1484047735, | |
| "size_mb": 1415.2982091903687, | |
| "size_gb": 1.3821271574124694, | |
| "path": "common_crawl/2023-06/en_middle_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1408488376, | |
| "size_mb": 1343.2391891479492, | |
| "size_gb": 1.3117570206522942, | |
| "path": "common_crawl/2019-30/en_middle_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0098.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1316646871, | |
| "size_mb": 1255.652304649353, | |
| "size_gb": 1.2262229537591338, | |
| "path": "common_crawl/2020-05/en_middle_0098.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0084.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1376502059, | |
| "size_mb": 1312.7346601486206, | |
| "size_gb": 1.2819674415513873, | |
| "path": "common_crawl/2020-05/en_middle_0084.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1436734425, | |
| "size_mb": 1370.1767206192017, | |
| "size_gb": 1.3380632037296891, | |
| "path": "common_crawl/2019-30/en_middle_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2046064401, | |
| "size_mb": 1951.2790689468384, | |
| "size_gb": 1.9055459657683969, | |
| "path": "common_crawl/2020-05/en_head_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0096.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1299464520, | |
| "size_mb": 1239.2659378051758, | |
| "size_gb": 1.210220642387867, | |
| "path": "common_crawl/2020-05/en_middle_0096.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1502071351, | |
| "size_mb": 1432.4868688583374, | |
| "size_gb": 1.3989129578694701, | |
| "path": "common_crawl/2023-06/en_middle_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2078296493, | |
| "size_mb": 1982.0179872512817, | |
| "size_gb": 1.9355644406750798, | |
| "path": "common_crawl/2022-05/en_head_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1999336324, | |
| "size_mb": 1906.7157020568848, | |
| "size_gb": 1.8620270527899265, | |
| "path": "common_crawl/2020-05/en_head_0061.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00892-of-01024.jsonl", | |
| "size": 847857590, | |
| "size_mb": 808.5800075531006, | |
| "size_gb": 0.7896289136260748, | |
| "path": "c4/c4-train.00892-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00952-of-01024.jsonl", | |
| "size": 845406836, | |
| "size_mb": 806.2427864074707, | |
| "size_gb": 0.7873464711010456, | |
| "path": "c4/c4-train.00952-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00959-of-01024.jsonl", | |
| "size": 847656085, | |
| "size_mb": 808.3878374099731, | |
| "size_gb": 0.7894412474706769, | |
| "path": "c4/c4-train.00959-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_fd572627-cce7-4667-a684-fef096dfbeb7.jsonl", | |
| "size": 899486705, | |
| "size_mb": 857.8173685073853, | |
| "size_gb": 0.8377122739329934, | |
| "path": "arxiv/arxiv_fd572627-cce7-4667-a684-fef096dfbeb7.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00830-of-01024.jsonl", | |
| "size": 846542448, | |
| "size_mb": 807.3257904052734, | |
| "size_gb": 0.7884040921926498, | |
| "path": "c4/c4-train.00830-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00951-of-01024.jsonl", | |
| "size": 843719089, | |
| "size_mb": 804.633225440979, | |
| "size_gb": 0.7857746342197061, | |
| "path": "c4/c4-train.00951-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1452824687, | |
| "size_mb": 1385.5215902328491, | |
| "size_gb": 1.3530484279617667, | |
| "path": "common_crawl/2019-30/en_middle_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00579-of-01024.jsonl", | |
| "size": 845510977, | |
| "size_mb": 806.3421030044556, | |
| "size_gb": 0.7874434599652886, | |
| "path": "c4/c4-train.00579-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00575-of-01024.jsonl", | |
| "size": 845610995, | |
| "size_mb": 806.4374876022339, | |
| "size_gb": 0.7875366089865565, | |
| "path": "c4/c4-train.00575-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00377-of-01024.jsonl", | |
| "size": 846089417, | |
| "size_mb": 806.8937463760376, | |
| "size_gb": 0.7879821741953492, | |
| "path": "c4/c4-train.00377-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00856-of-01024.jsonl", | |
| "size": 846578844, | |
| "size_mb": 807.3605003356934, | |
| "size_gb": 0.7884379886090755, | |
| "path": "c4/c4-train.00856-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00322-of-01024.jsonl", | |
| "size": 847505830, | |
| "size_mb": 808.2445430755615, | |
| "size_gb": 0.789301311597228, | |
| "path": "c4/c4-train.00322-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00396-of-01024.jsonl", | |
| "size": 848631346, | |
| "size_mb": 809.3179187774658, | |
| "size_gb": 0.790349530056119, | |
| "path": "c4/c4-train.00396-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00576-of-01024.jsonl", | |
| "size": 844473303, | |
| "size_mb": 805.352499961853, | |
| "size_gb": 0.7864770507439971, | |
| "path": "c4/c4-train.00576-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_a647274d-799d-4e7a-a485-b8632a87061e.jsonl", | |
| "size": 998657265, | |
| "size_mb": 952.3937845230103, | |
| "size_gb": 0.9300720551982522, | |
| "path": "arxiv/arxiv_a647274d-799d-4e7a-a485-b8632a87061e.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1535891745, | |
| "size_mb": 1464.7405099868774, | |
| "size_gb": 1.43041065428406, | |
| "path": "common_crawl/2023-06/en_middle_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00191-of-01024.jsonl", | |
| "size": 844350057, | |
| "size_mb": 805.2349634170532, | |
| "size_gb": 0.786362268961966, | |
| "path": "c4/c4-train.00191-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1497125573, | |
| "size_mb": 1427.7702074050903, | |
| "size_gb": 1.3943068431690335, | |
| "path": "common_crawl/2023-06/en_middle_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_8509f5a7-64a8-4813-92dc-f6eb53e3aacc.jsonl", | |
| "size": 994852744, | |
| "size_mb": 948.765510559082, | |
| "size_gb": 0.9265288189053535, | |
| "path": "arxiv/arxiv_8509f5a7-64a8-4813-92dc-f6eb53e3aacc.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00635-of-01024.jsonl", | |
| "size": 849998470, | |
| "size_mb": 810.6217098236084, | |
| "size_gb": 0.7916227634996176, | |
| "path": "c4/c4-train.00635-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00359-of-01024.jsonl", | |
| "size": 841959432, | |
| "size_mb": 802.9550857543945, | |
| "size_gb": 0.7841358259320259, | |
| "path": "c4/c4-train.00359-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00836-of-01024.jsonl", | |
| "size": 842420914, | |
| "size_mb": 803.3951892852783, | |
| "size_gb": 0.7845656145364046, | |
| "path": "c4/c4-train.00836-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00004-of-01024.jsonl", | |
| "size": 844566113, | |
| "size_mb": 805.4410104751587, | |
| "size_gb": 0.7865634867921472, | |
| "path": "c4/c4-train.00004-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00838-of-01024.jsonl", | |
| "size": 847776223, | |
| "size_mb": 808.5024099349976, | |
| "size_gb": 0.789553134702146, | |
| "path": "c4/c4-train.00838-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00351-of-01024.jsonl", | |
| "size": 845839405, | |
| "size_mb": 806.6553163528442, | |
| "size_gb": 0.7877493323758245, | |
| "path": "c4/c4-train.00351-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_fbc62949-624d-4943-9731-f5c46242ba55.jsonl", | |
| "size": 817494855, | |
| "size_mb": 779.6238470077515, | |
| "size_gb": 0.7613514130935073, | |
| "path": "arxiv/arxiv_fbc62949-624d-4943-9731-f5c46242ba55.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00837-of-01024.jsonl", | |
| "size": 849557042, | |
| "size_mb": 810.2007312774658, | |
| "size_gb": 0.7912116516381502, | |
| "path": "c4/c4-train.00837-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00363-of-01024.jsonl", | |
| "size": 844196761, | |
| "size_mb": 805.0887689590454, | |
| "size_gb": 0.7862195009365678, | |
| "path": "c4/c4-train.00363-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0095.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1424483502, | |
| "size_mb": 1358.493330001831, | |
| "size_gb": 1.3266536425799131, | |
| "path": "common_crawl/2019-30/en_middle_0095.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00840-of-01024.jsonl", | |
| "size": 847936337, | |
| "size_mb": 808.6551065444946, | |
| "size_gb": 0.789702252484858, | |
| "path": "c4/c4-train.00840-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00010-of-01024.jsonl", | |
| "size": 845613797, | |
| "size_mb": 806.4401597976685, | |
| "size_gb": 0.7875392185524106, | |
| "path": "c4/c4-train.00010-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0070.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1294402682, | |
| "size_mb": 1234.4385929107666, | |
| "size_gb": 1.2055064383894205, | |
| "path": "common_crawl/2020-05/en_middle_0070.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00842-of-01024.jsonl", | |
| "size": 844941459, | |
| "size_mb": 805.7989683151245, | |
| "size_gb": 0.7869130549952388, | |
| "path": "c4/c4-train.00842-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0075.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1513047999, | |
| "size_mb": 1442.9550161361694, | |
| "size_gb": 1.409135757945478, | |
| "path": "common_crawl/2023-06/en_middle_0075.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00843-of-01024.jsonl", | |
| "size": 845551836, | |
| "size_mb": 806.3810691833496, | |
| "size_gb": 0.7874815128743649, | |
| "path": "c4/c4-train.00843-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00001-of-01024.jsonl", | |
| "size": 843249487, | |
| "size_mb": 804.185378074646, | |
| "size_gb": 0.7853372832760215, | |
| "path": "c4/c4-train.00001-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00844-of-01024.jsonl", | |
| "size": 846609807, | |
| "size_mb": 807.3900289535522, | |
| "size_gb": 0.7884668251499534, | |
| "path": "c4/c4-train.00844-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00848-of-01024.jsonl", | |
| "size": 844052441, | |
| "size_mb": 804.9511346817017, | |
| "size_gb": 0.7860850924625993, | |
| "path": "c4/c4-train.00848-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00368-of-01024.jsonl", | |
| "size": 843119717, | |
| "size_mb": 804.061619758606, | |
| "size_gb": 0.7852164255455136, | |
| "path": "c4/c4-train.00368-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_c6e57a16-4879-4dcf-b591-503cfb46a360.jsonl", | |
| "size": 825974511, | |
| "size_mb": 787.7106771469116, | |
| "size_gb": 0.7692487081512809, | |
| "path": "arxiv/arxiv_c6e57a16-4879-4dcf-b591-503cfb46a360.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00847-of-01024.jsonl", | |
| "size": 842437950, | |
| "size_mb": 803.4114360809326, | |
| "size_gb": 0.7845814805477858, | |
| "path": "c4/c4-train.00847-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00360-of-01024.jsonl", | |
| "size": 848385576, | |
| "size_mb": 809.0835342407227, | |
| "size_gb": 0.7901206389069557, | |
| "path": "c4/c4-train.00360-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00839-of-01024.jsonl", | |
| "size": 844680987, | |
| "size_mb": 805.5505628585815, | |
| "size_gb": 0.7866704715415835, | |
| "path": "c4/c4-train.00839-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00849-of-01024.jsonl", | |
| "size": 846832275, | |
| "size_mb": 807.6021909713745, | |
| "size_gb": 0.7886740146204829, | |
| "path": "c4/c4-train.00849-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0077.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1422503801, | |
| "size_mb": 1356.6053400039673, | |
| "size_gb": 1.3248099023476243, | |
| "path": "common_crawl/2019-30/en_middle_0077.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00850-of-01024.jsonl", | |
| "size": 845133702, | |
| "size_mb": 805.9823055267334, | |
| "size_gb": 0.7870920952409506, | |
| "path": "c4/c4-train.00850-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_d6a7fc44-b584-4dd8-9de2-e981afe0bb4a.jsonl", | |
| "size": 839795968, | |
| "size_mb": 800.891845703125, | |
| "size_gb": 0.782120943069458, | |
| "path": "arxiv/arxiv_d6a7fc44-b584-4dd8-9de2-e981afe0bb4a.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00369-of-01024.jsonl", | |
| "size": 847232424, | |
| "size_mb": 807.9838027954102, | |
| "size_gb": 0.7890466824173927, | |
| "path": "c4/c4-train.00369-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_d41edf9f-0ebb-4866-b3fe-50785746b36b.jsonl", | |
| "size": 1059020618, | |
| "size_mb": 1009.9607639312744, | |
| "size_gb": 0.9862898085266352, | |
| "path": "arxiv/arxiv_d41edf9f-0ebb-4866-b3fe-50785746b36b.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/book/book.jsonl", | |
| "size": 107837638842, | |
| "size_mb": 102841.98650550842, | |
| "size_gb": 100.43162744678557, | |
| "path": "book/book.jsonl", | |
| "top_folder": "book", | |
| "top_folders": "book" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00362-of-01024.jsonl", | |
| "size": 845661120, | |
| "size_mb": 806.4852905273438, | |
| "size_gb": 0.7875832915306091, | |
| "path": "c4/c4-train.00362-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00364-of-01024.jsonl", | |
| "size": 844760437, | |
| "size_mb": 805.62633228302, | |
| "size_gb": 0.7867444651201367, | |
| "path": "c4/c4-train.00364-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1503146500, | |
| "size_mb": 1433.5122108459473, | |
| "size_gb": 1.3999142684042454, | |
| "path": "common_crawl/2023-06/en_middle_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00356-of-01024.jsonl", | |
| "size": 844519096, | |
| "size_mb": 805.3961715698242, | |
| "size_gb": 0.7865196987986565, | |
| "path": "c4/c4-train.00356-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_c05da42b-4939-4f55-867c-16cf6d228e60.jsonl", | |
| "size": 958895005, | |
| "size_mb": 914.4735383987427, | |
| "size_gb": 0.8930405648425221, | |
| "path": "arxiv/arxiv_c05da42b-4939-4f55-867c-16cf6d228e60.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_94a393a1-3b23-4961-a0a6-70bad5b4979c.jsonl", | |
| "size": 969500789, | |
| "size_mb": 924.588002204895, | |
| "size_gb": 0.9029179709032178, | |
| "path": "arxiv/arxiv_94a393a1-3b23-4961-a0a6-70bad5b4979c.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00003-of-01024.jsonl", | |
| "size": 845031731, | |
| "size_mb": 805.8850584030151, | |
| "size_gb": 0.7869971273466945, | |
| "path": "c4/c4-train.00003-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_98e718fd-5b0e-439f-a00c-57b61e06b395.jsonl", | |
| "size": 1084350290, | |
| "size_mb": 1034.117021560669, | |
| "size_gb": 1.0098799038678408, | |
| "path": "arxiv/arxiv_98e718fd-5b0e-439f-a00c-57b61e06b395.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00355-of-01024.jsonl", | |
| "size": 845603642, | |
| "size_mb": 806.4304752349854, | |
| "size_gb": 0.7875297609716654, | |
| "path": "c4/c4-train.00355-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00845-of-01024.jsonl", | |
| "size": 846605705, | |
| "size_mb": 807.3861169815063, | |
| "size_gb": 0.7884630048647523, | |
| "path": "c4/c4-train.00845-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00846-of-01024.jsonl", | |
| "size": 843026489, | |
| "size_mb": 803.972710609436, | |
| "size_gb": 0.7851296002045274, | |
| "path": "c4/c4-train.00846-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_9f8d7d10-dda7-4e44-b00c-811635a199c8.jsonl", | |
| "size": 964924202, | |
| "size_mb": 920.2234287261963, | |
| "size_gb": 0.8986556921154261, | |
| "path": "arxiv/arxiv_9f8d7d10-dda7-4e44-b00c-811635a199c8.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00834-of-01024.jsonl", | |
| "size": 843821531, | |
| "size_mb": 804.7309217453003, | |
| "size_gb": 0.7858700407668948, | |
| "path": "c4/c4-train.00834-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00366-of-01024.jsonl", | |
| "size": 847132846, | |
| "size_mb": 807.888837814331, | |
| "size_gb": 0.7889539431780577, | |
| "path": "c4/c4-train.00366-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00370-of-01024.jsonl", | |
| "size": 843007421, | |
| "size_mb": 803.9545259475708, | |
| "size_gb": 0.7851118417456746, | |
| "path": "c4/c4-train.00370-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00367-of-01024.jsonl", | |
| "size": 846184338, | |
| "size_mb": 806.9842700958252, | |
| "size_gb": 0.7880705762654543, | |
| "path": "c4/c4-train.00367-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00006-of-01024.jsonl", | |
| "size": 844028235, | |
| "size_mb": 804.9280500411987, | |
| "size_gb": 0.7860625488683581, | |
| "path": "c4/c4-train.00006-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00835-of-01024.jsonl", | |
| "size": 847718740, | |
| "size_mb": 808.4475898742676, | |
| "size_gb": 0.7894995994865894, | |
| "path": "c4/c4-train.00835-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00365-of-01024.jsonl", | |
| "size": 845680234, | |
| "size_mb": 806.5035190582275, | |
| "size_gb": 0.7876010928303003, | |
| "path": "c4/c4-train.00365-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0079.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1499660790, | |
| "size_mb": 1430.1879787445068, | |
| "size_gb": 1.3966679479926825, | |
| "path": "common_crawl/2023-06/en_middle_0079.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1567530521, | |
| "size_mb": 1494.913598060608, | |
| "size_gb": 1.4598765606060624, | |
| "path": "common_crawl/2023-06/en_middle_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00008-of-01024.jsonl", | |
| "size": 843263669, | |
| "size_mb": 804.1989030838013, | |
| "size_gb": 0.7853504912927747, | |
| "path": "c4/c4-train.00008-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1503691201, | |
| "size_mb": 1434.031678199768, | |
| "size_gb": 1.400421560741961, | |
| "path": "common_crawl/2023-06/en_middle_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00803-of-01024.jsonl", | |
| "size": 846184154, | |
| "size_mb": 806.984094619751, | |
| "size_gb": 0.7880704049021006, | |
| "path": "c4/c4-train.00803-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1527991905, | |
| "size_mb": 1457.2066354751587, | |
| "size_gb": 1.4230533549562097, | |
| "path": "common_crawl/2023-06/en_middle_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_3f9eb7ad-f266-4154-8d4d-54deeffde075.jsonl", | |
| "size": 896954421, | |
| "size_mb": 855.4023942947388, | |
| "size_gb": 0.8353539006784558, | |
| "path": "arxiv/arxiv_3f9eb7ad-f266-4154-8d4d-54deeffde075.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00371-of-01024.jsonl", | |
| "size": 845486509, | |
| "size_mb": 806.3187685012817, | |
| "size_gb": 0.787420672364533, | |
| "path": "c4/c4-train.00371-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00101-of-01024.jsonl", | |
| "size": 845817388, | |
| "size_mb": 806.6343193054199, | |
| "size_gb": 0.7877288274466991, | |
| "path": "c4/c4-train.00101-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00806-of-01024.jsonl", | |
| "size": 847282670, | |
| "size_mb": 808.0317211151123, | |
| "size_gb": 0.7890934776514769, | |
| "path": "c4/c4-train.00806-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00053-of-01024.jsonl", | |
| "size": 850109308, | |
| "size_mb": 810.7274131774902, | |
| "size_gb": 0.7917259894311428, | |
| "path": "c4/c4-train.00053-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1515935152, | |
| "size_mb": 1445.7084197998047, | |
| "size_gb": 1.4118246287107468, | |
| "path": "common_crawl/2023-06/en_middle_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1519430474, | |
| "size_mb": 1449.0418186187744, | |
| "size_gb": 1.4150799009948969, | |
| "path": "common_crawl/2023-06/en_middle_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00555-of-01024.jsonl", | |
| "size": 849944584, | |
| "size_mb": 810.5703201293945, | |
| "size_gb": 0.7915725782513618, | |
| "path": "c4/c4-train.00555-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00093-of-01024.jsonl", | |
| "size": 846057726, | |
| "size_mb": 806.8635234832764, | |
| "size_gb": 0.7879526596516371, | |
| "path": "c4/c4-train.00093-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0090.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1499886516, | |
| "size_mb": 1430.403247833252, | |
| "size_gb": 1.39687817171216, | |
| "path": "common_crawl/2023-06/en_middle_0090.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0076.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1530840526, | |
| "size_mb": 1459.9232921600342, | |
| "size_gb": 1.4257063400000334, | |
| "path": "common_crawl/2023-06/en_middle_0076.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00072-of-01024.jsonl", | |
| "size": 844949664, | |
| "size_mb": 805.8067932128906, | |
| "size_gb": 0.7869206964969635, | |
| "path": "c4/c4-train.00072-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00183-of-01024.jsonl", | |
| "size": 847694507, | |
| "size_mb": 808.4244794845581, | |
| "size_gb": 0.7894770307466388, | |
| "path": "c4/c4-train.00183-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1490675372, | |
| "size_mb": 1421.6188163757324, | |
| "size_gb": 1.3882996253669262, | |
| "path": "common_crawl/2023-06/en_middle_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_023827cd-7ee8-42e6-aa7b-661731f4c70f.jsonl", | |
| "size": 810555210, | |
| "size_mb": 773.0056858062744, | |
| "size_gb": 0.7548883650451899, | |
| "path": "arxiv/arxiv_023827cd-7ee8-42e6-aa7b-661731f4c70f.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0081.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1533800740, | |
| "size_mb": 1462.7463722229004, | |
| "size_gb": 1.4284632541239262, | |
| "path": "common_crawl/2023-06/en_middle_0081.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00080-of-01024.jsonl", | |
| "size": 848506099, | |
| "size_mb": 809.1984739303589, | |
| "size_gb": 0.7902328846976161, | |
| "path": "c4/c4-train.00080-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0084.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1504957523, | |
| "size_mb": 1435.2393369674683, | |
| "size_gb": 1.4016009150072932, | |
| "path": "common_crawl/2023-06/en_middle_0084.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0089.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1510198975, | |
| "size_mb": 1440.2379751205444, | |
| "size_gb": 1.4064823975786567, | |
| "path": "common_crawl/2023-06/en_middle_0089.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0098.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1496835148, | |
| "size_mb": 1427.493236541748, | |
| "size_gb": 1.3940363638103008, | |
| "path": "common_crawl/2023-06/en_middle_0098.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1537004514, | |
| "size_mb": 1465.8017292022705, | |
| "size_gb": 1.4314470011740923, | |
| "path": "common_crawl/2023-06/en_middle_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_575ae832-e770-4a89-bfa7-c56f16dbca69.jsonl", | |
| "size": 839258611, | |
| "size_mb": 800.3793821334839, | |
| "size_gb": 0.7816204903647304, | |
| "path": "arxiv/arxiv_575ae832-e770-4a89-bfa7-c56f16dbca69.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00852-of-01024.jsonl", | |
| "size": 847407696, | |
| "size_mb": 808.1509552001953, | |
| "size_gb": 0.7892099171876907, | |
| "path": "c4/c4-train.00852-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00308-of-01024.jsonl", | |
| "size": 843835043, | |
| "size_mb": 804.7438077926636, | |
| "size_gb": 0.785882624797523, | |
| "path": "c4/c4-train.00308-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00073-of-01024.jsonl", | |
| "size": 842927828, | |
| "size_mb": 803.8786201477051, | |
| "size_gb": 0.7850377149879932, | |
| "path": "c4/c4-train.00073-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00789-of-01024.jsonl", | |
| "size": 842922914, | |
| "size_mb": 803.8739337921143, | |
| "size_gb": 0.7850331384688616, | |
| "path": "c4/c4-train.00789-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00038-of-01024.jsonl", | |
| "size": 843820033, | |
| "size_mb": 804.7294931411743, | |
| "size_gb": 0.785868645645678, | |
| "path": "c4/c4-train.00038-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00527-of-01024.jsonl", | |
| "size": 840441158, | |
| "size_mb": 801.5071468353271, | |
| "size_gb": 0.7827218230813742, | |
| "path": "c4/c4-train.00527-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00851-of-01024.jsonl", | |
| "size": 846478921, | |
| "size_mb": 807.2652063369751, | |
| "size_gb": 0.7883449280634522, | |
| "path": "c4/c4-train.00851-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00041-of-01024.jsonl", | |
| "size": 846659372, | |
| "size_mb": 807.4372978210449, | |
| "size_gb": 0.7885129861533642, | |
| "path": "c4/c4-train.00041-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1803088695, | |
| "size_mb": 1719.5593786239624, | |
| "size_gb": 1.6792572056874633, | |
| "path": "common_crawl/2021-04/en_head_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1501138348, | |
| "size_mb": 1431.5970878601074, | |
| "size_gb": 1.3980440311133862, | |
| "path": "common_crawl/2023-06/en_middle_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1498413016, | |
| "size_mb": 1428.9980087280273, | |
| "size_gb": 1.3955058678984642, | |
| "path": "common_crawl/2023-06/en_middle_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1492323623, | |
| "size_mb": 1423.1907110214233, | |
| "size_gb": 1.3898346787318587, | |
| "path": "common_crawl/2023-06/en_middle_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1519083054, | |
| "size_mb": 1448.7104930877686, | |
| "size_gb": 1.414756340906024, | |
| "path": "common_crawl/2023-06/en_middle_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00528-of-01024.jsonl", | |
| "size": 844151793, | |
| "size_mb": 805.0458841323853, | |
| "size_gb": 0.7861776212230325, | |
| "path": "c4/c4-train.00528-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1284048068, | |
| "size_mb": 1224.563663482666, | |
| "size_gb": 1.195862952619791, | |
| "path": "common_crawl/2020-05/en_middle_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0083.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1511299895, | |
| "size_mb": 1441.2878942489624, | |
| "size_gb": 1.4075077092275023, | |
| "path": "common_crawl/2023-06/en_middle_0083.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0092.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1412404911, | |
| "size_mb": 1346.9742879867554, | |
| "size_gb": 1.3154045781120658, | |
| "path": "common_crawl/2023-06/en_middle_0092.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1513095512, | |
| "size_mb": 1443.0003280639648, | |
| "size_gb": 1.4091800078749657, | |
| "path": "common_crawl/2023-06/en_middle_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0085.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1502209111, | |
| "size_mb": 1432.6182470321655, | |
| "size_gb": 1.3990412568673491, | |
| "path": "common_crawl/2023-06/en_middle_0085.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0087.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1494822915, | |
| "size_mb": 1425.574221611023, | |
| "size_gb": 1.3921623257920146, | |
| "path": "common_crawl/2023-06/en_middle_0087.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1499883809, | |
| "size_mb": 1430.4006662368774, | |
| "size_gb": 1.3968756506219506, | |
| "path": "common_crawl/2023-06/en_middle_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0077.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1506020454, | |
| "size_mb": 1436.2530269622803, | |
| "size_gb": 1.4025908466428518, | |
| "path": "common_crawl/2023-06/en_middle_0077.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1530364430, | |
| "size_mb": 1459.4692516326904, | |
| "size_gb": 1.4252629410475492, | |
| "path": "common_crawl/2023-06/en_middle_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0122.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1324704710, | |
| "size_mb": 1263.3368587493896, | |
| "size_gb": 1.2337274011224508, | |
| "path": "common_crawl/2020-05/en_middle_0122.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1486278204, | |
| "size_mb": 1417.425350189209, | |
| "size_gb": 1.3842044435441494, | |
| "path": "common_crawl/2023-06/en_middle_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1489211361, | |
| "size_mb": 1420.2226266860962, | |
| "size_gb": 1.3869361588731408, | |
| "path": "common_crawl/2023-06/en_middle_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0086.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1549706439, | |
| "size_mb": 1477.915228843689, | |
| "size_gb": 1.443276590667665, | |
| "path": "common_crawl/2023-06/en_middle_0086.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00853-of-01024.jsonl", | |
| "size": 841447654, | |
| "size_mb": 802.4670162200928, | |
| "size_gb": 0.7836591955274343, | |
| "path": "c4/c4-train.00853-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00559-of-01024.jsonl", | |
| "size": 848965522, | |
| "size_mb": 809.6366138458252, | |
| "size_gb": 0.7906607557088137, | |
| "path": "c4/c4-train.00559-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00119-of-01024.jsonl", | |
| "size": 844737907, | |
| "size_mb": 805.6048460006714, | |
| "size_gb": 0.7867234824225307, | |
| "path": "c4/c4-train.00119-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00557-of-01024.jsonl", | |
| "size": 845901770, | |
| "size_mb": 806.7147922515869, | |
| "size_gb": 0.7878074143081903, | |
| "path": "c4/c4-train.00557-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00124-of-01024.jsonl", | |
| "size": 842034136, | |
| "size_mb": 803.0263290405273, | |
| "size_gb": 0.78420539945364, | |
| "path": "c4/c4-train.00124-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_c6b43cda-ca5c-4855-9c08-0f8264cab1af.jsonl", | |
| "size": 898151565, | |
| "size_mb": 856.5440797805786, | |
| "size_gb": 0.8364688279107213, | |
| "path": "arxiv/arxiv_c6b43cda-ca5c-4855-9c08-0f8264cab1af.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00558-of-01024.jsonl", | |
| "size": 846962889, | |
| "size_mb": 807.7267541885376, | |
| "size_gb": 0.7887956583872437, | |
| "path": "c4/c4-train.00558-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2152737439, | |
| "size_mb": 2053.0104055404663, | |
| "size_gb": 2.0048929741606116, | |
| "path": "common_crawl/2019-30/en_head_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00373-of-01024.jsonl", | |
| "size": 848085428, | |
| "size_mb": 808.797290802002, | |
| "size_gb": 0.78984110429883, | |
| "path": "c4/c4-train.00373-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0082.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1525373461, | |
| "size_mb": 1454.7094926834106, | |
| "size_gb": 1.4206147389486432, | |
| "path": "common_crawl/2023-06/en_middle_0082.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00116-of-01024.jsonl", | |
| "size": 845791126, | |
| "size_mb": 806.6092739105225, | |
| "size_gb": 0.7877043690532446, | |
| "path": "c4/c4-train.00116-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0121.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1328338695, | |
| "size_mb": 1266.8024969100952, | |
| "size_gb": 1.2371118133887649, | |
| "path": "common_crawl/2020-05/en_middle_0121.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1864075012, | |
| "size_mb": 1777.7204627990723, | |
| "size_gb": 1.736055139452219, | |
| "path": "common_crawl/2021-04/en_head_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1451695779, | |
| "size_mb": 1384.4449796676636, | |
| "size_gb": 1.3519970504567027, | |
| "path": "common_crawl/2019-30/en_middle_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0088.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1502336693, | |
| "size_mb": 1432.7399187088013, | |
| "size_gb": 1.3991600768640637, | |
| "path": "common_crawl/2023-06/en_middle_0088.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0080.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1488147681, | |
| "size_mb": 1419.2082223892212, | |
| "size_gb": 1.3859455296769738, | |
| "path": "common_crawl/2023-06/en_middle_0080.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0102.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1290329237, | |
| "size_mb": 1230.5538530349731, | |
| "size_gb": 1.201712747104466, | |
| "path": "common_crawl/2020-05/en_middle_0102.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1549335580, | |
| "size_mb": 1477.5615501403809, | |
| "size_gb": 1.4429312013089657, | |
| "path": "common_crawl/2023-06/en_middle_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00593-of-01024.jsonl", | |
| "size": 851055733, | |
| "size_mb": 811.629994392395, | |
| "size_gb": 0.7926074163988233, | |
| "path": "c4/c4-train.00593-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00788-of-01024.jsonl", | |
| "size": 846552785, | |
| "size_mb": 807.3356485366821, | |
| "size_gb": 0.7884137192741036, | |
| "path": "c4/c4-train.00788-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00384-of-01024.jsonl", | |
| "size": 844082049, | |
| "size_mb": 804.9793710708618, | |
| "size_gb": 0.7861126670613885, | |
| "path": "c4/c4-train.00384-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1820455684, | |
| "size_mb": 1736.1218299865723, | |
| "size_gb": 1.695431474596262, | |
| "path": "common_crawl/2021-04/en_head_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00190-of-01024.jsonl", | |
| "size": 844130587, | |
| "size_mb": 805.0256605148315, | |
| "size_gb": 0.7861578715965152, | |
| "path": "c4/c4-train.00190-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00414-of-01024.jsonl", | |
| "size": 841005148, | |
| "size_mb": 802.0450096130371, | |
| "size_gb": 0.7832470797002316, | |
| "path": "c4/c4-train.00414-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00417-of-01024.jsonl", | |
| "size": 847632272, | |
| "size_mb": 808.3651275634766, | |
| "size_gb": 0.7894190698862076, | |
| "path": "c4/c4-train.00417-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_bf11ef29-a3f9-4a2d-9dbf-ebcc56d39fdb.jsonl", | |
| "size": 898510414, | |
| "size_mb": 856.8863048553467, | |
| "size_gb": 0.8368030320852995, | |
| "path": "arxiv/arxiv_bf11ef29-a3f9-4a2d-9dbf-ebcc56d39fdb.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00657-of-01024.jsonl", | |
| "size": 843778233, | |
| "size_mb": 804.6896295547485, | |
| "size_gb": 0.7858297163620591, | |
| "path": "c4/c4-train.00657-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00383-of-01024.jsonl", | |
| "size": 846456688, | |
| "size_mb": 807.2440032958984, | |
| "size_gb": 0.7883242219686508, | |
| "path": "c4/c4-train.00383-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00656-of-01024.jsonl", | |
| "size": 849087356, | |
| "size_mb": 809.7528038024902, | |
| "size_gb": 0.7907742224633694, | |
| "path": "c4/c4-train.00656-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00597-of-01024.jsonl", | |
| "size": 845670505, | |
| "size_mb": 806.4942407608032, | |
| "size_gb": 0.7875920319929719, | |
| "path": "c4/c4-train.00597-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00375-of-01024.jsonl", | |
| "size": 846070130, | |
| "size_mb": 806.8753528594971, | |
| "size_gb": 0.7879642117768526, | |
| "path": "c4/c4-train.00375-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_eb4e26d4-6625-4f8a-b5fe-6f3a9b8a4b79.jsonl", | |
| "size": 967664186, | |
| "size_mb": 922.8364810943604, | |
| "size_gb": 0.9012075010687113, | |
| "path": "arxiv/arxiv_eb4e26d4-6625-4f8a-b5fe-6f3a9b8a4b79.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00007-of-01024.jsonl", | |
| "size": 844054851, | |
| "size_mb": 804.9534330368042, | |
| "size_gb": 0.7860873369500041, | |
| "path": "c4/c4-train.00007-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00374-of-01024.jsonl", | |
| "size": 845778275, | |
| "size_mb": 806.5970182418823, | |
| "size_gb": 0.7876924006268382, | |
| "path": "c4/c4-train.00374-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00857-of-01024.jsonl", | |
| "size": 844215288, | |
| "size_mb": 805.1064376831055, | |
| "size_gb": 0.7862367555499077, | |
| "path": "c4/c4-train.00857-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1857138977, | |
| "size_mb": 1771.1057443618774, | |
| "size_gb": 1.729595453478396, | |
| "path": "common_crawl/2021-04/en_head_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1513287745, | |
| "size_mb": 1443.1836557388306, | |
| "size_gb": 1.4093590388074517, | |
| "path": "common_crawl/2023-06/en_middle_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00382-of-01024.jsonl", | |
| "size": 847813618, | |
| "size_mb": 808.5380725860596, | |
| "size_gb": 0.7895879615098238, | |
| "path": "c4/c4-train.00382-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00653-of-01024.jsonl", | |
| "size": 847090758, | |
| "size_mb": 807.8486995697021, | |
| "size_gb": 0.7889147456735373, | |
| "path": "c4/c4-train.00653-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00380-of-01024.jsonl", | |
| "size": 847924562, | |
| "size_mb": 808.643877029419, | |
| "size_gb": 0.7896912861615419, | |
| "path": "c4/c4-train.00380-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_a1e3430e-ef5c-4a86-914d-88e8fb7818c0.jsonl", | |
| "size": 827579171, | |
| "size_mb": 789.2410001754761, | |
| "size_gb": 0.7707431642338634, | |
| "path": "arxiv/arxiv_a1e3430e-ef5c-4a86-914d-88e8fb7818c0.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00858-of-01024.jsonl", | |
| "size": 844370149, | |
| "size_mb": 805.2541246414185, | |
| "size_gb": 0.7863809810951352, | |
| "path": "c4/c4-train.00858-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_b0732cff-657e-4e69-87b8-66e8025bf441.jsonl", | |
| "size": 950652584, | |
| "size_mb": 906.6129531860352, | |
| "size_gb": 0.8853642120957375, | |
| "path": "arxiv/arxiv_b0732cff-657e-4e69-87b8-66e8025bf441.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_c1fc3dd5-861f-4b8d-b7a2-eb8f6887b33b.jsonl", | |
| "size": 952542650, | |
| "size_mb": 908.4154605865479, | |
| "size_gb": 0.8871244732290506, | |
| "path": "arxiv/arxiv_c1fc3dd5-861f-4b8d-b7a2-eb8f6887b33b.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00379-of-01024.jsonl", | |
| "size": 849485125, | |
| "size_mb": 810.1321458816528, | |
| "size_gb": 0.7911446737125516, | |
| "path": "c4/c4-train.00379-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00862-of-01024.jsonl", | |
| "size": 844704661, | |
| "size_mb": 805.5731401443481, | |
| "size_gb": 0.786692519672215, | |
| "path": "c4/c4-train.00862-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00596-of-01024.jsonl", | |
| "size": 844184857, | |
| "size_mb": 805.0774164199829, | |
| "size_gb": 0.7862084144726396, | |
| "path": "c4/c4-train.00596-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_caf769e4-7308-4419-9114-900ca213682a.jsonl", | |
| "size": 1002407231, | |
| "size_mb": 955.9700307846069, | |
| "size_gb": 0.9335644831880927, | |
| "path": "arxiv/arxiv_caf769e4-7308-4419-9114-900ca213682a.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00854-of-01024.jsonl", | |
| "size": 847184455, | |
| "size_mb": 807.9380559921265, | |
| "size_gb": 0.789002007804811, | |
| "path": "c4/c4-train.00854-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_08cdfa755e6d4d89b673d5bd1acee5f6.sampled.jsonl", | |
| "size": 2658456835, | |
| "size_mb": 2535.302004814148, | |
| "size_gb": 2.4758808640763164, | |
| "path": "github/filtered_08cdfa755e6d4d89b673d5bd1acee5f6.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_ad160286662849f49d1e6de27c0f1d15.sampled.jsonl", | |
| "size": 2540070637, | |
| "size_mb": 2422.400128364563, | |
| "size_gb": 2.3656251253560185, | |
| "path": "github/filtered_ad160286662849f49d1e6de27c0f1d15.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00416-of-01024.jsonl", | |
| "size": 847020809, | |
| "size_mb": 807.7819910049438, | |
| "size_gb": 0.7888496005907655, | |
| "path": "c4/c4-train.00416-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_ab7c0034-7fc1-4fa8-bae3-e97b85fc16a4.jsonl", | |
| "size": 736040313, | |
| "size_mb": 701.9427423477173, | |
| "size_gb": 0.6854909593239427, | |
| "path": "arxiv/arxiv_ab7c0034-7fc1-4fa8-bae3-e97b85fc16a4.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_d7c49dbb-c008-47fc-9cbe-8d5695842d21.jsonl", | |
| "size": 942270734, | |
| "size_mb": 898.6193981170654, | |
| "size_gb": 0.8775580059736967, | |
| "path": "arxiv/arxiv_d7c49dbb-c008-47fc-9cbe-8d5695842d21.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00860-of-01024.jsonl", | |
| "size": 847683811, | |
| "size_mb": 808.4142789840698, | |
| "size_gb": 0.7894670693203807, | |
| "path": "c4/c4-train.00860-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_a2b4cb3d-bea3-478e-82a2-77c00a827250.jsonl", | |
| "size": 953409095, | |
| "size_mb": 909.2417669296265, | |
| "size_gb": 0.8879314130172133, | |
| "path": "arxiv/arxiv_a2b4cb3d-bea3-478e-82a2-77c00a827250.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_db981f69-9eca-4031-8565-318b949efbfe.jsonl", | |
| "size": 853069613, | |
| "size_mb": 813.5505800247192, | |
| "size_gb": 0.7944829883053899, | |
| "path": "arxiv/arxiv_db981f69-9eca-4031-8565-318b949efbfe.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00381-of-01024.jsonl", | |
| "size": 846229188, | |
| "size_mb": 807.027042388916, | |
| "size_gb": 0.7881123460829258, | |
| "path": "c4/c4-train.00381-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00207-of-01024.jsonl", | |
| "size": 840884208, | |
| "size_mb": 801.9296722412109, | |
| "size_gb": 0.7831344455480576, | |
| "path": "c4/c4-train.00207-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00859-of-01024.jsonl", | |
| "size": 846695985, | |
| "size_mb": 807.4722146987915, | |
| "size_gb": 0.7885470846667886, | |
| "path": "c4/c4-train.00859-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0097.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1528106766, | |
| "size_mb": 1457.3161754608154, | |
| "size_gb": 1.4231603275984526, | |
| "path": "common_crawl/2023-06/en_middle_0097.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1831350030, | |
| "size_mb": 1746.5114879608154, | |
| "size_gb": 1.7055776249617338, | |
| "path": "common_crawl/2021-04/en_head_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00372-of-01024.jsonl", | |
| "size": 842658700, | |
| "size_mb": 803.6219596862793, | |
| "size_gb": 0.7847870700061321, | |
| "path": "c4/c4-train.00372-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00378-of-01024.jsonl", | |
| "size": 846426561, | |
| "size_mb": 807.2152719497681, | |
| "size_gb": 0.7882961640134454, | |
| "path": "c4/c4-train.00378-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00855-of-01024.jsonl", | |
| "size": 845685834, | |
| "size_mb": 806.5088596343994, | |
| "size_gb": 0.7876063082367182, | |
| "path": "c4/c4-train.00855-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00376-of-01024.jsonl", | |
| "size": 844307622, | |
| "size_mb": 805.1944942474365, | |
| "size_gb": 0.7863227482885122, | |
| "path": "c4/c4-train.00376-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2050410732, | |
| "size_mb": 1955.4240531921387, | |
| "size_gb": 1.909593801945448, | |
| "path": "common_crawl/2020-05/en_head_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00863-of-01024.jsonl", | |
| "size": 846470886, | |
| "size_mb": 807.2575435638428, | |
| "size_gb": 0.7883374448865652, | |
| "path": "c4/c4-train.00863-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00861-of-01024.jsonl", | |
| "size": 844467787, | |
| "size_mb": 805.3472394943237, | |
| "size_gb": 0.7864719135686755, | |
| "path": "c4/c4-train.00861-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0093.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1478360314, | |
| "size_mb": 1409.874261856079, | |
| "size_gb": 1.3768303338438272, | |
| "path": "common_crawl/2023-06/en_middle_0093.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00204-of-01024.jsonl", | |
| "size": 841354639, | |
| "size_mb": 802.3783102035522, | |
| "size_gb": 0.7835725685581565, | |
| "path": "c4/c4-train.00204-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0104.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 257807850, | |
| "size_mb": 245.86472511291504, | |
| "size_gb": 0.2401022706180811, | |
| "path": "common_crawl/2023-06/en_middle_0104.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1977668011, | |
| "size_mb": 1886.051188468933, | |
| "size_gb": 1.8418468637391925, | |
| "path": "common_crawl/2020-05/en_head_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1513833232, | |
| "size_mb": 1443.703872680664, | |
| "size_gb": 1.409867063164711, | |
| "path": "common_crawl/2023-06/en_middle_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0100.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1497272738, | |
| "size_mb": 1427.9105548858643, | |
| "size_gb": 1.3944439012557268, | |
| "path": "common_crawl/2023-06/en_middle_0100.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00865-of-01024.jsonl", | |
| "size": 846888913, | |
| "size_mb": 807.6562051773071, | |
| "size_gb": 0.788726762868464, | |
| "path": "c4/c4-train.00865-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_10f129bfd0af45caa9cd72aa9d863ec5.sampled.jsonl", | |
| "size": 1612287311, | |
| "size_mb": 1537.596999168396, | |
| "size_gb": 1.5015595695003867, | |
| "path": "github/filtered_10f129bfd0af45caa9cd72aa9d863ec5.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00387-of-01024.jsonl", | |
| "size": 846594282, | |
| "size_mb": 807.37522315979, | |
| "size_gb": 0.7884523663669825, | |
| "path": "c4/c4-train.00387-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1283339617, | |
| "size_mb": 1223.8880319595337, | |
| "size_gb": 1.1952031562104821, | |
| "path": "common_crawl/2020-05/en_middle_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1519417307, | |
| "size_mb": 1449.0292615890503, | |
| "size_gb": 1.415067638270557, | |
| "path": "common_crawl/2023-06/en_middle_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00272-of-01024.jsonl", | |
| "size": 845087153, | |
| "size_mb": 805.937912940979, | |
| "size_gb": 0.7870487431064248, | |
| "path": "c4/c4-train.00272-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00386-of-01024.jsonl", | |
| "size": 845863259, | |
| "size_mb": 806.6780652999878, | |
| "size_gb": 0.7877715481445193, | |
| "path": "c4/c4-train.00386-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00866-of-01024.jsonl", | |
| "size": 847602155, | |
| "size_mb": 808.3364057540894, | |
| "size_gb": 0.7893910212442279, | |
| "path": "c4/c4-train.00866-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_ae1813abc63f4b1998dfa608e7fe5588.sampled.jsonl", | |
| "size": 2098064934, | |
| "size_mb": 2000.870641708374, | |
| "size_gb": 1.953975236043334, | |
| "path": "github/filtered_ae1813abc63f4b1998dfa608e7fe5588.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00274-of-01024.jsonl", | |
| "size": 843801345, | |
| "size_mb": 804.7116708755493, | |
| "size_gb": 0.7858512410894036, | |
| "path": "c4/c4-train.00274-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_aabc2d54d1c946908d3400228e0f238c.sampled.jsonl", | |
| "size": 1798019097, | |
| "size_mb": 1714.724633216858, | |
| "size_gb": 1.6745357746258378, | |
| "path": "github/filtered_aabc2d54d1c946908d3400228e0f238c.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00991-of-01024.jsonl", | |
| "size": 845516452, | |
| "size_mb": 806.3473243713379, | |
| "size_gb": 0.7874485589563847, | |
| "path": "c4/c4-train.00991-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00389-of-01024.jsonl", | |
| "size": 846838031, | |
| "size_mb": 807.6076803207397, | |
| "size_gb": 0.7886793753132224, | |
| "path": "c4/c4-train.00389-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0094.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1484047301, | |
| "size_mb": 1415.2977952957153, | |
| "size_gb": 1.382126753218472, | |
| "path": "common_crawl/2023-06/en_middle_0094.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_3d5a011a-4bbe-4585-a2bd-ff3e943c8671.jsonl", | |
| "size": 920973489, | |
| "size_mb": 878.308762550354, | |
| "size_gb": 0.8577234009280801, | |
| "path": "arxiv/arxiv_3d5a011a-4bbe-4585-a2bd-ff3e943c8671.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00869-of-01024.jsonl", | |
| "size": 847272554, | |
| "size_mb": 808.0220737457275, | |
| "size_gb": 0.789084056392312, | |
| "path": "c4/c4-train.00869-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_7d1d69dc-bc8e-4817-9cab-afdc002ab7c4.jsonl", | |
| "size": 806577722, | |
| "size_mb": 769.2124576568604, | |
| "size_gb": 0.7511840406805277, | |
| "path": "arxiv/arxiv_7d1d69dc-bc8e-4817-9cab-afdc002ab7c4.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_29d54f5a-1dd0-4e9a-b783-fb2eec9db072.jsonl", | |
| "size": 871803706, | |
| "size_mb": 831.4168033599854, | |
| "size_gb": 0.8119304720312357, | |
| "path": "arxiv/arxiv_29d54f5a-1dd0-4e9a-b783-fb2eec9db072.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00385-of-01024.jsonl", | |
| "size": 847153820, | |
| "size_mb": 807.9088401794434, | |
| "size_gb": 0.7889734767377377, | |
| "path": "c4/c4-train.00385-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_2b224cd9-286e-46ac-8c4e-c1e3befc8760.jsonl", | |
| "size": 950016447, | |
| "size_mb": 906.0062856674194, | |
| "size_gb": 0.8847717633470893, | |
| "path": "arxiv/arxiv_2b224cd9-286e-46ac-8c4e-c1e3befc8760.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_adda41d791974289aff042e2e3d07ec3.sampled.jsonl", | |
| "size": 1891344953, | |
| "size_mb": 1803.727105140686, | |
| "size_gb": 1.7614522511139512, | |
| "path": "github/filtered_adda41d791974289aff042e2e3d07ec3.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00275-of-01024.jsonl", | |
| "size": 843989490, | |
| "size_mb": 804.8910999298096, | |
| "size_gb": 0.7860264647752047, | |
| "path": "c4/c4-train.00275-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_1d3a0cd6-f0e6-4106-a080-524a4bd50016.jsonl", | |
| "size": 1179702376, | |
| "size_mb": 1125.051856994629, | |
| "size_gb": 1.0986834540963173, | |
| "path": "arxiv/arxiv_1d3a0cd6-f0e6-4106-a080-524a4bd50016.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00271-of-01024.jsonl", | |
| "size": 847214525, | |
| "size_mb": 807.9667329788208, | |
| "size_gb": 0.7890300126746297, | |
| "path": "c4/c4-train.00271-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00753-of-01024.jsonl", | |
| "size": 851285756, | |
| "size_mb": 811.8493614196777, | |
| "size_gb": 0.792821642011404, | |
| "path": "c4/c4-train.00753-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1403593900, | |
| "size_mb": 1338.5714530944824, | |
| "size_gb": 1.3071986846625805, | |
| "path": "common_crawl/2023-06/en_middle_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00994-of-01024.jsonl", | |
| "size": 849148447, | |
| "size_mb": 809.8110647201538, | |
| "size_gb": 0.7908311178907752, | |
| "path": "c4/c4-train.00994-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00864-of-01024.jsonl", | |
| "size": 843006310, | |
| "size_mb": 803.9534664154053, | |
| "size_gb": 0.7851108070462942, | |
| "path": "c4/c4-train.00864-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00752-of-01024.jsonl", | |
| "size": 849924543, | |
| "size_mb": 810.5512075424194, | |
| "size_gb": 0.791553913615644, | |
| "path": "c4/c4-train.00752-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_af7b386db97e4211a7378be08d7b3f4f.sampled.jsonl", | |
| "size": 2482340465, | |
| "size_mb": 2367.3443460464478, | |
| "size_gb": 2.311859712935984, | |
| "path": "github/filtered_af7b386db97e4211a7378be08d7b3f4f.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00276-of-01024.jsonl", | |
| "size": 844449523, | |
| "size_mb": 805.3298215866089, | |
| "size_gb": 0.7864549038931727, | |
| "path": "c4/c4-train.00276-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00755-of-01024.jsonl", | |
| "size": 841934678, | |
| "size_mb": 802.9314785003662, | |
| "size_gb": 0.7841127719730139, | |
| "path": "c4/c4-train.00755-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_0f27d10d846a473b96070c3394832f32.sampled.jsonl", | |
| "size": 1584645656, | |
| "size_mb": 1511.2358627319336, | |
| "size_gb": 1.475816272199154, | |
| "path": "github/filtered_0f27d10d846a473b96070c3394832f32.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_0f979046c8e64e0fb5843d2634a9957d.sampled.jsonl", | |
| "size": 1931024379, | |
| "size_mb": 1841.5683546066284, | |
| "size_gb": 1.7984065962955356, | |
| "path": "github/filtered_0f979046c8e64e0fb5843d2634a9957d.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00992-of-01024.jsonl", | |
| "size": 843042429, | |
| "size_mb": 803.9879121780396, | |
| "size_gb": 0.7851444454863667, | |
| "path": "c4/c4-train.00992-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0096.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1518768259, | |
| "size_mb": 1448.4102811813354, | |
| "size_gb": 1.414463165216148, | |
| "path": "common_crawl/2023-06/en_middle_0096.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00754-of-01024.jsonl", | |
| "size": 846159553, | |
| "size_mb": 806.9606332778931, | |
| "size_gb": 0.7880474934354424, | |
| "path": "c4/c4-train.00754-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1524238141, | |
| "size_mb": 1453.6267671585083, | |
| "size_gb": 1.4195573898032308, | |
| "path": "common_crawl/2023-06/en_middle_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00496-of-01024.jsonl", | |
| "size": 841581649, | |
| "size_mb": 802.5948038101196, | |
| "size_gb": 0.78378398809582, | |
| "path": "c4/c4-train.00496-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00756-of-01024.jsonl", | |
| "size": 840875385, | |
| "size_mb": 801.9212579727173, | |
| "size_gb": 0.7831262284889817, | |
| "path": "c4/c4-train.00756-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1503042939, | |
| "size_mb": 1433.413447380066, | |
| "size_gb": 1.3998178197070956, | |
| "path": "common_crawl/2023-06/en_middle_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00996-of-01024.jsonl", | |
| "size": 849514534, | |
| "size_mb": 810.160192489624, | |
| "size_gb": 0.7911720629781485, | |
| "path": "c4/c4-train.00996-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1496764408, | |
| "size_mb": 1427.4257736206055, | |
| "size_gb": 1.3939704820513725, | |
| "path": "common_crawl/2023-06/en_middle_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00751-of-01024.jsonl", | |
| "size": 842573836, | |
| "size_mb": 803.5410270690918, | |
| "size_gb": 0.78470803424716, | |
| "path": "c4/c4-train.00751-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_b15c575fa9f8465d98f70ba2f2f73c6e.sampled.jsonl", | |
| "size": 1643654237, | |
| "size_mb": 1567.5108308792114, | |
| "size_gb": 1.53077229578048, | |
| "path": "github/filtered_b15c575fa9f8465d98f70ba2f2f73c6e.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_9f50a028-2586-4e0d-bcfd-d9d2d74e8953.jsonl", | |
| "size": 758313102, | |
| "size_mb": 723.1837291717529, | |
| "size_gb": 0.70623411051929, | |
| "path": "arxiv/arxiv_9f50a028-2586-4e0d-bcfd-d9d2d74e8953.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00993-of-01024.jsonl", | |
| "size": 846159855, | |
| "size_mb": 806.9609212875366, | |
| "size_gb": 0.78804777469486, | |
| "path": "c4/c4-train.00993-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00144-of-01024.jsonl", | |
| "size": 848388021, | |
| "size_mb": 809.0858659744263, | |
| "size_gb": 0.7901229159906507, | |
| "path": "c4/c4-train.00144-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00273-of-01024.jsonl", | |
| "size": 844144036, | |
| "size_mb": 805.0384864807129, | |
| "size_gb": 0.7861703969538212, | |
| "path": "c4/c4-train.00273-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00995-of-01024.jsonl", | |
| "size": 842301852, | |
| "size_mb": 803.2816429138184, | |
| "size_gb": 0.7844547294080257, | |
| "path": "c4/c4-train.00995-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1523515586, | |
| "size_mb": 1452.9376850128174, | |
| "size_gb": 1.4188844580203295, | |
| "path": "common_crawl/2023-06/en_middle_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1425581406, | |
| "size_mb": 1359.5403728485107, | |
| "size_gb": 1.3276761453598738, | |
| "path": "common_crawl/2019-30/en_middle_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2075611861, | |
| "size_mb": 1979.4577226638794, | |
| "size_gb": 1.9330641822889447, | |
| "path": "common_crawl/2020-05/en_head_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1481962245, | |
| "size_mb": 1413.3093309402466, | |
| "size_gb": 1.3801848934963346, | |
| "path": "common_crawl/2023-06/en_middle_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1527194258, | |
| "size_mb": 1456.4459400177002, | |
| "size_gb": 1.4223104882985353, | |
| "path": "common_crawl/2023-06/en_middle_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_f7680c03-70df-4781-a98d-c88695f92f04.jsonl", | |
| "size": 971336103, | |
| "size_mb": 926.3382940292358, | |
| "size_gb": 0.9046272402629256, | |
| "path": "arxiv/arxiv_f7680c03-70df-4781-a98d-c88695f92f04.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0101.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1503375591, | |
| "size_mb": 1433.730689048767, | |
| "size_gb": 1.4001276260241866, | |
| "path": "common_crawl/2023-06/en_middle_0101.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00403-of-01024.jsonl", | |
| "size": 842326226, | |
| "size_mb": 803.3048877716064, | |
| "size_gb": 0.7844774294644594, | |
| "path": "c4/c4-train.00403-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2016536545, | |
| "size_mb": 1923.1191110610962, | |
| "size_gb": 1.8780460068956017, | |
| "path": "common_crawl/2020-05/en_head_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00643-of-01024.jsonl", | |
| "size": 841528199, | |
| "size_mb": 802.5438299179077, | |
| "size_gb": 0.7837342089042068, | |
| "path": "c4/c4-train.00643-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00200-of-01024.jsonl", | |
| "size": 842173291, | |
| "size_mb": 803.1590375900269, | |
| "size_gb": 0.7843349976465106, | |
| "path": "c4/c4-train.00200-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1377017549, | |
| "size_mb": 1313.2262697219849, | |
| "size_gb": 1.2824475290253758, | |
| "path": "common_crawl/2019-30/en_middle_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2003944729, | |
| "size_mb": 1911.110619544983, | |
| "size_gb": 1.8663189643993974, | |
| "path": "common_crawl/2020-05/en_head_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00583-of-01024.jsonl", | |
| "size": 846069533, | |
| "size_mb": 806.8747835159302, | |
| "size_gb": 0.7879636557772756, | |
| "path": "c4/c4-train.00583-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2136007303, | |
| "size_mb": 2037.0553045272827, | |
| "size_gb": 1.9893118208274245, | |
| "path": "common_crawl/2019-30/en_head_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1362929673, | |
| "size_mb": 1299.7910242080688, | |
| "size_gb": 1.2693271720781922, | |
| "path": "common_crawl/2019-30/en_middle_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00920-of-01024.jsonl", | |
| "size": 846533546, | |
| "size_mb": 807.3173007965088, | |
| "size_gb": 0.7883958015590906, | |
| "path": "c4/c4-train.00920-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00303-of-01024.jsonl", | |
| "size": 847180174, | |
| "size_mb": 807.9339733123779, | |
| "size_gb": 0.7889980208128691, | |
| "path": "c4/c4-train.00303-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_8ede1b71-6846-439a-acba-86a57cfec3d2.jsonl", | |
| "size": 995554970, | |
| "size_mb": 949.4352054595947, | |
| "size_gb": 0.9271828178316355, | |
| "path": "arxiv/arxiv_8ede1b71-6846-439a-acba-86a57cfec3d2.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00785-of-01024.jsonl", | |
| "size": 845472772, | |
| "size_mb": 806.3056678771973, | |
| "size_gb": 0.7874078787863255, | |
| "path": "c4/c4-train.00785-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00923-of-01024.jsonl", | |
| "size": 847618227, | |
| "size_mb": 808.3517332077026, | |
| "size_gb": 0.7894059894606471, | |
| "path": "c4/c4-train.00923-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00132-of-01024.jsonl", | |
| "size": 842729216, | |
| "size_mb": 803.689208984375, | |
| "size_gb": 0.7848527431488037, | |
| "path": "c4/c4-train.00132-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00980-of-01024.jsonl", | |
| "size": 844467163, | |
| "size_mb": 805.3466444015503, | |
| "size_gb": 0.786471332423389, | |
| "path": "c4/c4-train.00980-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00309-of-01024.jsonl", | |
| "size": 843344525, | |
| "size_mb": 804.2760133743286, | |
| "size_gb": 0.7854257943108678, | |
| "path": "c4/c4-train.00309-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00924-of-01024.jsonl", | |
| "size": 847489870, | |
| "size_mb": 808.2293224334717, | |
| "size_gb": 0.7892864476889372, | |
| "path": "c4/c4-train.00924-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0099.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1521526944, | |
| "size_mb": 1451.0411682128906, | |
| "size_gb": 1.417032390832901, | |
| "path": "common_crawl/2023-06/en_middle_0099.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00787-of-01024.jsonl", | |
| "size": 844077278, | |
| "size_mb": 804.9748210906982, | |
| "size_gb": 0.786108223721385, | |
| "path": "c4/c4-train.00787-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00982-of-01024.jsonl", | |
| "size": 848504540, | |
| "size_mb": 809.1969871520996, | |
| "size_gb": 0.7902314327657223, | |
| "path": "c4/c4-train.00982-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00305-of-01024.jsonl", | |
| "size": 843237212, | |
| "size_mb": 804.1736717224121, | |
| "size_gb": 0.7853258512914181, | |
| "path": "c4/c4-train.00305-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00981-of-01024.jsonl", | |
| "size": 848832837, | |
| "size_mb": 809.5100755691528, | |
| "size_gb": 0.7905371831730008, | |
| "path": "c4/c4-train.00981-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00922-of-01024.jsonl", | |
| "size": 846626806, | |
| "size_mb": 807.4062404632568, | |
| "size_gb": 0.7884826567023993, | |
| "path": "c4/c4-train.00922-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01023-of-01024.jsonl", | |
| "size": 843079160, | |
| "size_mb": 804.0229415893555, | |
| "size_gb": 0.785178653895855, | |
| "path": "c4/c4-train.01023-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00307-of-01024.jsonl", | |
| "size": 845660910, | |
| "size_mb": 806.4850902557373, | |
| "size_gb": 0.7875830959528685, | |
| "path": "c4/c4-train.00307-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1482557071, | |
| "size_mb": 1413.8766012191772, | |
| "size_gb": 1.3807388683781028, | |
| "path": "common_crawl/2023-06/en_middle_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00159-of-01024.jsonl", | |
| "size": 845739183, | |
| "size_mb": 806.5597372055054, | |
| "size_gb": 0.7876559933647513, | |
| "path": "c4/c4-train.00159-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00157-of-01024.jsonl", | |
| "size": 848143758, | |
| "size_mb": 808.8529186248779, | |
| "size_gb": 0.7898954283446074, | |
| "path": "c4/c4-train.00157-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0095.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1519898065, | |
| "size_mb": 1449.4877481460571, | |
| "size_gb": 1.415515379048884, | |
| "path": "common_crawl/2023-06/en_middle_0095.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1365992671, | |
| "size_mb": 1302.7121267318726, | |
| "size_gb": 1.2721798112615943, | |
| "path": "common_crawl/2019-30/en_middle_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1406384345, | |
| "size_mb": 1341.2326288223267, | |
| "size_gb": 1.3097974890843034, | |
| "path": "common_crawl/2019-30/en_middle_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00404-of-01024.jsonl", | |
| "size": 850465193, | |
| "size_mb": 811.0668115615845, | |
| "size_gb": 0.7920574331656098, | |
| "path": "c4/c4-train.00404-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00921-of-01024.jsonl", | |
| "size": 846460944, | |
| "size_mb": 807.2480621337891, | |
| "size_gb": 0.7883281856775284, | |
| "path": "c4/c4-train.00921-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00983-of-01024.jsonl", | |
| "size": 846148059, | |
| "size_mb": 806.9496717453003, | |
| "size_gb": 0.7880367888137698, | |
| "path": "c4/c4-train.00983-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1980006493, | |
| "size_mb": 1888.2813386917114, | |
| "size_gb": 1.8440247448161244, | |
| "path": "common_crawl/2020-05/en_head_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1290424746, | |
| "size_mb": 1230.6449375152588, | |
| "size_gb": 1.201801696792245, | |
| "path": "common_crawl/2020-05/en_middle_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00979-of-01024.jsonl", | |
| "size": 844551621, | |
| "size_mb": 805.4271898269653, | |
| "size_gb": 0.7865499900653958, | |
| "path": "c4/c4-train.00979-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0103.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1534617130, | |
| "size_mb": 1463.5249423980713, | |
| "size_gb": 1.4292235765606165, | |
| "path": "common_crawl/2023-06/en_middle_0103.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0102.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1478439925, | |
| "size_mb": 1409.9501848220825, | |
| "size_gb": 1.376904477365315, | |
| "path": "common_crawl/2023-06/en_middle_0102.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2044459617, | |
| "size_mb": 1949.7486276626587, | |
| "size_gb": 1.9040513942018151, | |
| "path": "common_crawl/2020-05/en_head_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1372045303, | |
| "size_mb": 1308.4843664169312, | |
| "size_gb": 1.2778167640790343, | |
| "path": "common_crawl/2020-05/en_middle_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2110041416, | |
| "size_mb": 2012.2923049926758, | |
| "size_gb": 1.96512920409441, | |
| "path": "common_crawl/2019-30/en_head_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2043781558, | |
| "size_mb": 1949.1019802093506, | |
| "size_gb": 1.903419902548194, | |
| "path": "common_crawl/2020-05/en_head_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2046652861, | |
| "size_mb": 1951.8402681350708, | |
| "size_gb": 1.906094011850655, | |
| "path": "common_crawl/2020-05/en_head_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2039724675, | |
| "size_mb": 1945.2330350875854, | |
| "size_gb": 1.8996416358277202, | |
| "path": "common_crawl/2020-05/en_head_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1353100262, | |
| "size_mb": 1290.4169673919678, | |
| "size_gb": 1.2601728197187185, | |
| "path": "common_crawl/2020-05/en_middle_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0092.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1309990481, | |
| "size_mb": 1249.3042764663696, | |
| "size_gb": 1.220023707486689, | |
| "path": "common_crawl/2020-05/en_middle_0092.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1257792565, | |
| "size_mb": 1199.5244646072388, | |
| "size_gb": 1.1714106099680066, | |
| "path": "common_crawl/2020-05/en_middle_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1431165575, | |
| "size_mb": 1364.8658514022827, | |
| "size_gb": 1.3328768080100417, | |
| "path": "common_crawl/2019-30/en_middle_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1390243102, | |
| "size_mb": 1325.8391399383545, | |
| "size_gb": 1.2947647850960493, | |
| "path": "common_crawl/2019-30/en_middle_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0073.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1373363484, | |
| "size_mb": 1309.7414817810059, | |
| "size_gb": 1.2790444158017635, | |
| "path": "common_crawl/2019-30/en_middle_0073.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1517281795, | |
| "size_mb": 1446.992678642273, | |
| "size_gb": 1.4130787877365947, | |
| "path": "common_crawl/2023-06/en_middle_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1280342048, | |
| "size_mb": 1221.0293273925781, | |
| "size_gb": 1.1924114525318146, | |
| "path": "common_crawl/2020-05/en_middle_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2163341294, | |
| "size_mb": 2063.1230297088623, | |
| "size_gb": 2.014768583700061, | |
| "path": "common_crawl/2019-30/en_head_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0091.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1346358068, | |
| "size_mb": 1283.9871101379395, | |
| "size_gb": 1.2538936622440815, | |
| "path": "common_crawl/2020-05/en_middle_0091.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2023-06/en_middle_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1505211859, | |
| "size_mb": 1435.4818906784058, | |
| "size_gb": 1.4018377838656306, | |
| "path": "common_crawl/2023-06/en_middle_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2023-06" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2116804441, | |
| "size_mb": 2018.7420282363892, | |
| "size_gb": 1.9714277619495988, | |
| "path": "common_crawl/2020-05/en_head_0059.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00784-of-01024.jsonl", | |
| "size": 848029891, | |
| "size_mb": 808.7443265914917, | |
| "size_gb": 0.7897893814370036, | |
| "path": "c4/c4-train.00784-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2183544752, | |
| "size_mb": 2082.3905487060547, | |
| "size_gb": 2.0335845202207565, | |
| "path": "common_crawl/2019-30/en_head_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00790-of-01024.jsonl", | |
| "size": 845649812, | |
| "size_mb": 806.4745063781738, | |
| "size_gb": 0.7875727601349354, | |
| "path": "c4/c4-train.00790-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_88f24f8d-16d3-4a21-894d-192033d0fa67.jsonl", | |
| "size": 979686947, | |
| "size_mb": 934.3022794723511, | |
| "size_gb": 0.9124045697972178, | |
| "path": "arxiv/arxiv_88f24f8d-16d3-4a21-894d-192033d0fa67.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_610c82ed-b9ee-449c-83b0-601205f3a74a.jsonl", | |
| "size": 865172328, | |
| "size_mb": 825.0926284790039, | |
| "size_gb": 0.8057545199990273, | |
| "path": "arxiv/arxiv_610c82ed-b9ee-449c-83b0-601205f3a74a.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2011257439, | |
| "size_mb": 1918.08456325531, | |
| "size_gb": 1.8731294563040137, | |
| "path": "common_crawl/2020-05/en_head_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00304-of-01024.jsonl", | |
| "size": 844860614, | |
| "size_mb": 805.7218685150146, | |
| "size_gb": 0.786837762221694, | |
| "path": "c4/c4-train.00304-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00791-of-01024.jsonl", | |
| "size": 846718897, | |
| "size_mb": 807.494065284729, | |
| "size_gb": 0.7885684231296182, | |
| "path": "c4/c4-train.00791-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00306-of-01024.jsonl", | |
| "size": 847164125, | |
| "size_mb": 807.9186677932739, | |
| "size_gb": 0.7889830740168691, | |
| "path": "c4/c4-train.00306-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00786-of-01024.jsonl", | |
| "size": 845643571, | |
| "size_mb": 806.4685544967651, | |
| "size_gb": 0.7875669477507472, | |
| "path": "c4/c4-train.00786-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00498-of-01024.jsonl", | |
| "size": 843784446, | |
| "size_mb": 804.6955547332764, | |
| "size_gb": 0.7858355026692152, | |
| "path": "c4/c4-train.00498-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00133-of-01024.jsonl", | |
| "size": 845451739, | |
| "size_mb": 806.2856092453003, | |
| "size_gb": 0.7873882902786136, | |
| "path": "c4/c4-train.00133-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00738-of-01024.jsonl", | |
| "size": 847992500, | |
| "size_mb": 808.708667755127, | |
| "size_gb": 0.7897545583546162, | |
| "path": "c4/c4-train.00738-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00311-of-01024.jsonl", | |
| "size": 847788171, | |
| "size_mb": 808.51380443573, | |
| "size_gb": 0.7895642621442676, | |
| "path": "c4/c4-train.00311-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_580be642-bb73-4d0d-8b5e-f494722934cd.jsonl", | |
| "size": 876625471, | |
| "size_mb": 836.0151968002319, | |
| "size_gb": 0.8164210906252265, | |
| "path": "arxiv/arxiv_580be642-bb73-4d0d-8b5e-f494722934cd.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00792-of-01024.jsonl", | |
| "size": 846561827, | |
| "size_mb": 807.3442716598511, | |
| "size_gb": 0.7884221402928233, | |
| "path": "c4/c4-train.00792-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00743-of-01024.jsonl", | |
| "size": 845175445, | |
| "size_mb": 806.0221147537231, | |
| "size_gb": 0.7871309714391828, | |
| "path": "c4/c4-train.00743-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1325966186, | |
| "size_mb": 1264.5398960113525, | |
| "size_gb": 1.2349022421985865, | |
| "path": "common_crawl/2020-05/en_middle_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_3f805f4b-6f7f-42a8-a006-47c1e0401bd7.jsonl", | |
| "size": 903437176, | |
| "size_mb": 861.584831237793, | |
| "size_gb": 0.8413914367556572, | |
| "path": "arxiv/arxiv_3f805f4b-6f7f-42a8-a006-47c1e0401bd7.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2240041594, | |
| "size_mb": 2136.2701358795166, | |
| "size_gb": 2.0862013045698404, | |
| "path": "common_crawl/2019-30/en_head_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2118847550, | |
| "size_mb": 2020.6904888153076, | |
| "size_gb": 1.9733305554836988, | |
| "path": "common_crawl/2022-05/en_head_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1317705180, | |
| "size_mb": 1256.6615867614746, | |
| "size_gb": 1.2272085808217525, | |
| "path": "common_crawl/2020-05/en_middle_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1412978375, | |
| "size_mb": 1347.521185874939, | |
| "size_gb": 1.315938658080995, | |
| "path": "common_crawl/2020-05/en_middle_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00782-of-01024.jsonl", | |
| "size": 845865146, | |
| "size_mb": 806.6798648834229, | |
| "size_gb": 0.7877733055502176, | |
| "path": "c4/c4-train.00782-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00763-of-01024.jsonl", | |
| "size": 844240026, | |
| "size_mb": 805.1300296783447, | |
| "size_gb": 0.7862597946077585, | |
| "path": "c4/c4-train.00763-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00450-of-01024.jsonl", | |
| "size": 842302517, | |
| "size_mb": 803.2822771072388, | |
| "size_gb": 0.7844553487375379, | |
| "path": "c4/c4-train.00450-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00114-of-01024.jsonl", | |
| "size": 845593916, | |
| "size_mb": 806.421199798584, | |
| "size_gb": 0.7875207029283047, | |
| "path": "c4/c4-train.00114-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0071.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1430319405, | |
| "size_mb": 1364.0588808059692, | |
| "size_gb": 1.3320887507870793, | |
| "path": "common_crawl/2019-30/en_middle_0071.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_034e819a-cfcb-43c6-ad25-0232ad48823c.jsonl", | |
| "size": 912709988, | |
| "size_mb": 870.4280738830566, | |
| "size_gb": 0.8500274159014225, | |
| "path": "arxiv/arxiv_034e819a-cfcb-43c6-ad25-0232ad48823c.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00052-of-01024.jsonl", | |
| "size": 851207794, | |
| "size_mb": 811.7750110626221, | |
| "size_gb": 0.7927490342408419, | |
| "path": "c4/c4-train.00052-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00757-of-01024.jsonl", | |
| "size": 847428324, | |
| "size_mb": 808.1706275939941, | |
| "size_gb": 0.7892291285097599, | |
| "path": "c4/c4-train.00757-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_b40763d0b9ce4e0d8fb5f519f1f49f8c.sampled.jsonl", | |
| "size": 1650316800, | |
| "size_mb": 1573.86474609375, | |
| "size_gb": 1.5369772911071777, | |
| "path": "github/filtered_b40763d0b9ce4e0d8fb5f519f1f49f8c.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00277-of-01024.jsonl", | |
| "size": 847238043, | |
| "size_mb": 807.989161491394, | |
| "size_gb": 0.7890519155189395, | |
| "path": "c4/c4-train.00277-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00313-of-01024.jsonl", | |
| "size": 846204398, | |
| "size_mb": 807.0034008026123, | |
| "size_gb": 0.7880892585963011, | |
| "path": "c4/c4-train.00313-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_39367d6c-d7d4-45fc-a929-8a17184d1744.jsonl", | |
| "size": 1006798277, | |
| "size_mb": 960.1576585769653, | |
| "size_gb": 0.9376539634540677, | |
| "path": "arxiv/arxiv_39367d6c-d7d4-45fc-a929-8a17184d1744.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2154480920, | |
| "size_mb": 2054.6731185913086, | |
| "size_gb": 2.006516717374325, | |
| "path": "common_crawl/2019-30/en_head_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2161450709, | |
| "size_mb": 2061.3200273513794, | |
| "size_gb": 2.0130078392103314, | |
| "path": "common_crawl/2019-30/en_head_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00793-of-01024.jsonl", | |
| "size": 845715396, | |
| "size_mb": 806.537052154541, | |
| "size_gb": 0.787633839994669, | |
| "path": "c4/c4-train.00793-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00522-of-01024.jsonl", | |
| "size": 849767522, | |
| "size_mb": 810.401460647583, | |
| "size_gb": 0.7914076764136553, | |
| "path": "c4/c4-train.00522-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00286-of-01024.jsonl", | |
| "size": 842378812, | |
| "size_mb": 803.355037689209, | |
| "size_gb": 0.7845264039933681, | |
| "path": "c4/c4-train.00286-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1566798269, | |
| "size_mb": 1494.2152681350708, | |
| "size_gb": 1.459194597788155, | |
| "path": "common_crawl/2021-04/en_middle_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00290-of-01024.jsonl", | |
| "size": 850705920, | |
| "size_mb": 811.29638671875, | |
| "size_gb": 0.7922816276550293, | |
| "path": "c4/c4-train.00290-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_03232e26-be3f-4a28-a5d2-ee1d8c0e9831.jsonl", | |
| "size": 988545121, | |
| "size_mb": 942.7500925064087, | |
| "size_gb": 0.9206543872132897, | |
| "path": "arxiv/arxiv_03232e26-be3f-4a28-a5d2-ee1d8c0e9831.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_0af50072-df4c-4084-a833-cebbd046e70e.jsonl", | |
| "size": 951030819, | |
| "size_mb": 906.9736661911011, | |
| "size_gb": 0.8857164708897471, | |
| "path": "arxiv/arxiv_0af50072-df4c-4084-a833-cebbd046e70e.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_d8d45c58819948c9b352d74383944c4a.sampled.jsonl", | |
| "size": 3835485374, | |
| "size_mb": 3657.80389213562, | |
| "size_gb": 3.5720741134136915, | |
| "path": "github/filtered_d8d45c58819948c9b352d74383944c4a.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_7b26046f-7c8d-405b-911b-df51e1a069fa.jsonl", | |
| "size": 1042857552, | |
| "size_mb": 994.5464630126953, | |
| "size_gb": 0.9712367802858353, | |
| "path": "arxiv/arxiv_7b26046f-7c8d-405b-911b-df51e1a069fa.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00767-of-01024.jsonl", | |
| "size": 844151294, | |
| "size_mb": 805.0454082489014, | |
| "size_gb": 0.7861771564930677, | |
| "path": "c4/c4-train.00767-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00285-of-01024.jsonl", | |
| "size": 844297012, | |
| "size_mb": 805.1843757629395, | |
| "size_gb": 0.7863128669559956, | |
| "path": "c4/c4-train.00285-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1429007291, | |
| "size_mb": 1362.8075513839722, | |
| "size_gb": 1.3308667493984103, | |
| "path": "common_crawl/2019-30/en_middle_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_cac864a472b948b0bfe18c8e9a19aeb5.sampled.jsonl", | |
| "size": 1707988796, | |
| "size_mb": 1628.865047454834, | |
| "size_gb": 1.5906885229051113, | |
| "path": "github/filtered_cac864a472b948b0bfe18c8e9a19aeb5.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00765-of-01024.jsonl", | |
| "size": 842471543, | |
| "size_mb": 803.4434728622437, | |
| "size_gb": 0.7846127664670348, | |
| "path": "c4/c4-train.00765-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00287-of-01024.jsonl", | |
| "size": 849452524, | |
| "size_mb": 810.1010551452637, | |
| "size_gb": 0.7911143116652966, | |
| "path": "c4/c4-train.00287-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00769-of-01024.jsonl", | |
| "size": 846674865, | |
| "size_mb": 807.452073097229, | |
| "size_gb": 0.7885274151340127, | |
| "path": "c4/c4-train.00769-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1316546862, | |
| "size_mb": 1255.5569286346436, | |
| "size_gb": 1.226129813119769, | |
| "path": "common_crawl/2020-05/en_middle_0018.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_d0a054a678fc4c38b496d10e91a2c735.sampled.jsonl", | |
| "size": 1627854135, | |
| "size_mb": 1552.4426794052124, | |
| "size_gb": 1.5160573041066527, | |
| "path": "github/filtered_d0a054a678fc4c38b496d10e91a2c735.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00997-of-01024.jsonl", | |
| "size": 846670192, | |
| "size_mb": 807.4476165771484, | |
| "size_gb": 0.7885230630636215, | |
| "path": "c4/c4-train.00997-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2158979846, | |
| "size_mb": 2058.963628768921, | |
| "size_gb": 2.0107066687196493, | |
| "path": "common_crawl/2019-30/en_head_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00770-of-01024.jsonl", | |
| "size": 847324329, | |
| "size_mb": 808.0714502334595, | |
| "size_gb": 0.7891322756186128, | |
| "path": "c4/c4-train.00770-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01006-of-01024.jsonl", | |
| "size": 849291337, | |
| "size_mb": 809.9473352432251, | |
| "size_gb": 0.790964194573462, | |
| "path": "c4/c4-train.01006-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_dda863d833614d04b96bbe21b161768d.sampled.jsonl", | |
| "size": 1931276549, | |
| "size_mb": 1841.8088426589966, | |
| "size_gb": 1.7986414479091763, | |
| "path": "github/filtered_dda863d833614d04b96bbe21b161768d.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1394722925, | |
| "size_mb": 1330.1114320755005, | |
| "size_gb": 1.298936945386231, | |
| "path": "common_crawl/2019-30/en_middle_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01007-of-01024.jsonl", | |
| "size": 844490440, | |
| "size_mb": 805.3688430786133, | |
| "size_gb": 0.7864930108189583, | |
| "path": "c4/c4-train.01007-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01010-of-01024.jsonl", | |
| "size": 844339253, | |
| "size_mb": 805.2246599197388, | |
| "size_gb": 0.7863522069528699, | |
| "path": "c4/c4-train.01010-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_cb330e8dc8ac411eba2dc8676c9c4403.sampled.jsonl", | |
| "size": 1857004814, | |
| "size_mb": 1770.9777965545654, | |
| "size_gb": 1.7294705044478178, | |
| "path": "github/filtered_cb330e8dc8ac411eba2dc8676c9c4403.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00302-of-01024.jsonl", | |
| "size": 848348273, | |
| "size_mb": 809.0479593276978, | |
| "size_gb": 0.7900858977809548, | |
| "path": "c4/c4-train.00302-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01005-of-01024.jsonl", | |
| "size": 848174559, | |
| "size_mb": 808.8822927474976, | |
| "size_gb": 0.7899241140112281, | |
| "path": "c4/c4-train.01005-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00713-of-01024.jsonl", | |
| "size": 847586185, | |
| "size_mb": 808.3211755752563, | |
| "size_gb": 0.7893761480227113, | |
| "path": "c4/c4-train.00713-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00766-of-01024.jsonl", | |
| "size": 842827811, | |
| "size_mb": 803.7832365036011, | |
| "size_gb": 0.7849445668980479, | |
| "path": "c4/c4-train.00766-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01022-of-01024.jsonl", | |
| "size": 849229876, | |
| "size_mb": 809.8887214660645, | |
| "size_gb": 0.7909069545567036, | |
| "path": "c4/c4-train.01022-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_5d8d402b-8277-480a-b5fa-71169726864f.jsonl", | |
| "size": 1112975768, | |
| "size_mb": 1061.416404724121, | |
| "size_gb": 1.0365394577383995, | |
| "path": "arxiv/arxiv_5d8d402b-8277-480a-b5fa-71169726864f.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00768-of-01024.jsonl", | |
| "size": 845573116, | |
| "size_mb": 806.4013633728027, | |
| "size_gb": 0.7875013314187527, | |
| "path": "c4/c4-train.00768-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00182-of-01024.jsonl", | |
| "size": 844513082, | |
| "size_mb": 805.3904361724854, | |
| "size_gb": 0.7865140978246927, | |
| "path": "c4/c4-train.00182-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_d33701435f964c90a86c22e204dd5fde.sampled.jsonl", | |
| "size": 1599455012, | |
| "size_mb": 1525.3591651916504, | |
| "size_gb": 1.489608559757471, | |
| "path": "github/filtered_d33701435f964c90a86c22e204dd5fde.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01009-of-01024.jsonl", | |
| "size": 847809906, | |
| "size_mb": 808.5345325469971, | |
| "size_gb": 0.7895845044404268, | |
| "path": "c4/c4-train.01009-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_7ea7a996-b1bb-4773-a36a-461dce2de861.jsonl", | |
| "size": 908104989, | |
| "size_mb": 866.0364046096802, | |
| "size_gb": 0.8457386763766408, | |
| "path": "arxiv/arxiv_7ea7a996-b1bb-4773-a36a-461dce2de861.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00473-of-01024.jsonl", | |
| "size": 846997135, | |
| "size_mb": 807.7594137191772, | |
| "size_gb": 0.788827552460134, | |
| "path": "c4/c4-train.00473-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1326126236, | |
| "size_mb": 1264.6925315856934, | |
| "size_gb": 1.2350513003766537, | |
| "path": "common_crawl/2020-05/en_middle_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_79d50803-f7d9-4aa8-bf1a-d807980a40c6.jsonl", | |
| "size": 946773946, | |
| "size_mb": 902.9139957427979, | |
| "size_gb": 0.881751948967576, | |
| "path": "arxiv/arxiv_79d50803-f7d9-4aa8-bf1a-d807980a40c6.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.01008-of-01024.jsonl", | |
| "size": 846496865, | |
| "size_mb": 807.2823190689087, | |
| "size_gb": 0.7883616397157311, | |
| "path": "c4/c4-train.01008-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00288-of-01024.jsonl", | |
| "size": 847341243, | |
| "size_mb": 808.0875806808472, | |
| "size_gb": 0.7891480280086398, | |
| "path": "c4/c4-train.00288-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_8232f276-9e3f-463a-9350-362de1b501d1.jsonl", | |
| "size": 838654729, | |
| "size_mb": 799.8034753799438, | |
| "size_gb": 0.7810580814257264, | |
| "path": "arxiv/arxiv_8232f276-9e3f-463a-9350-362de1b501d1.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_d148857715424aabbd32b8ffe56c4082.sampled.jsonl", | |
| "size": 1798661738, | |
| "size_mb": 1715.3375034332275, | |
| "size_gb": 1.6751342806965113, | |
| "path": "github/filtered_d148857715424aabbd32b8ffe56c4082.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00289-of-01024.jsonl", | |
| "size": 845435144, | |
| "size_mb": 806.2697830200195, | |
| "size_gb": 0.7873728349804878, | |
| "path": "c4/c4-train.00289-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1013402371, | |
| "size_mb": 966.455813407898, | |
| "size_gb": 0.9438045052811503, | |
| "path": "common_crawl/2020-05/en_head_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/github/filtered_d448c2553f474193a1224df1c38f74d4.sampled.jsonl", | |
| "size": 2892286886, | |
| "size_mb": 2758.2997188568115, | |
| "size_gb": 2.693652069196105, | |
| "path": "github/filtered_d448c2553f474193a1224df1c38f74d4.sampled.jsonl", | |
| "top_folder": "github", | |
| "top_folders": "github" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2157127911, | |
| "size_mb": 2057.197485923767, | |
| "size_gb": 2.008981919847429, | |
| "path": "common_crawl/2019-30/en_head_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/wikipedia/wiki.jsonl", | |
| "size": 120142320713, | |
| "size_mb": 114576.64557743073, | |
| "size_gb": 111.89125544670969, | |
| "path": "wikipedia/wiki.jsonl", | |
| "top_folder": "wikipedia", | |
| "top_folders": "wikipedia" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1419516402, | |
| "size_mb": 1353.7563343048096, | |
| "size_gb": 1.3220276702195406, | |
| "path": "common_crawl/2019-30/en_middle_0063.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00033-of-01024.jsonl", | |
| "size": 848732448, | |
| "size_mb": 809.4143371582031, | |
| "size_gb": 0.7904436886310577, | |
| "path": "c4/c4-train.00033-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1374718796, | |
| "size_mb": 1311.034008026123, | |
| "size_gb": 1.2803066484630108, | |
| "path": "common_crawl/2019-30/en_middle_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00113-of-01024.jsonl", | |
| "size": 843079125, | |
| "size_mb": 804.0229082107544, | |
| "size_gb": 0.7851786212995648, | |
| "path": "c4/c4-train.00113-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1389422813, | |
| "size_mb": 1325.056851387024, | |
| "size_gb": 1.2940008314326406, | |
| "path": "common_crawl/2019-30/en_middle_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1284943572, | |
| "size_mb": 1225.417682647705, | |
| "size_gb": 1.1966969557106495, | |
| "path": "common_crawl/2020-05/en_middle_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2015892895, | |
| "size_mb": 1922.5052785873413, | |
| "size_gb": 1.8774465611204505, | |
| "path": "common_crawl/2020-05/en_head_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00794-of-01024.jsonl", | |
| "size": 847242771, | |
| "size_mb": 807.993670463562, | |
| "size_gb": 0.7890563188120723, | |
| "path": "c4/c4-train.00794-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00551-of-01024.jsonl", | |
| "size": 845838756, | |
| "size_mb": 806.6546974182129, | |
| "size_gb": 0.7877487279474735, | |
| "path": "c4/c4-train.00551-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1358241969, | |
| "size_mb": 1295.320481300354, | |
| "size_gb": 1.264961407519877, | |
| "path": "common_crawl/2019-30/en_middle_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_8f74f6ba-1c53-42d5-a3c7-e4ef46a71133.jsonl", | |
| "size": 796839329, | |
| "size_mb": 759.9252023696899, | |
| "size_gb": 0.7421144554391503, | |
| "path": "arxiv/arxiv_8f74f6ba-1c53-42d5-a3c7-e4ef46a71133.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2082805381, | |
| "size_mb": 1986.317997932434, | |
| "size_gb": 1.9397636698558927, | |
| "path": "common_crawl/2020-05/en_head_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1382886718, | |
| "size_mb": 1318.8235454559326, | |
| "size_gb": 1.2879136186093092, | |
| "path": "common_crawl/2019-30/en_middle_0062.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2234449333, | |
| "size_mb": 2130.9369401931763, | |
| "size_gb": 2.0809931056573987, | |
| "path": "common_crawl/2022-05/en_head_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1422097009, | |
| "size_mb": 1356.2173929214478, | |
| "size_gb": 1.3244310477748513, | |
| "path": "common_crawl/2019-30/en_middle_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1399257678, | |
| "size_mb": 1334.4361095428467, | |
| "size_gb": 1.3031602632254362, | |
| "path": "common_crawl/2019-30/en_middle_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1806598552, | |
| "size_mb": 1722.906639099121, | |
| "size_gb": 1.6825260147452354, | |
| "path": "common_crawl/2021-04/en_head_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2151337199, | |
| "size_mb": 2051.6750326156616, | |
| "size_gb": 2.003588899038732, | |
| "path": "common_crawl/2022-05/en_head_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0070.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1390719470, | |
| "size_mb": 1326.2934398651123, | |
| "size_gb": 1.2952084373682737, | |
| "path": "common_crawl/2019-30/en_middle_0070.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2187715496, | |
| "size_mb": 2086.36808013916, | |
| "size_gb": 2.0374688282608986, | |
| "path": "common_crawl/2019-30/en_head_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1404719149, | |
| "size_mb": 1339.6445741653442, | |
| "size_gb": 1.308246654458344, | |
| "path": "common_crawl/2019-30/en_middle_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_29fd3d99-53fb-43e2-a4a5-2fd01bf77258.jsonl", | |
| "size": 1038006182, | |
| "size_mb": 989.9198360443115, | |
| "size_gb": 0.966718589887023, | |
| "path": "arxiv/arxiv_29fd3d99-53fb-43e2-a4a5-2fd01bf77258.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1329514132, | |
| "size_mb": 1267.9234809875488, | |
| "size_gb": 1.2382065244019032, | |
| "path": "common_crawl/2020-05/en_middle_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0076.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1386154901, | |
| "size_mb": 1321.9403276443481, | |
| "size_gb": 1.2909573512151837, | |
| "path": "common_crawl/2019-30/en_middle_0076.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1810630226, | |
| "size_mb": 1726.751543045044, | |
| "size_gb": 1.6862808037549257, | |
| "path": "common_crawl/2021-04/en_head_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1321157727, | |
| "size_mb": 1259.95419216156, | |
| "size_gb": 1.2304240157827735, | |
| "path": "common_crawl/2020-05/en_middle_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1309409633, | |
| "size_mb": 1248.7503366470337, | |
| "size_gb": 1.2194827506318688, | |
| "path": "common_crawl/2020-05/en_middle_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1345603232, | |
| "size_mb": 1283.2672424316406, | |
| "size_gb": 1.253190666437149, | |
| "path": "common_crawl/2020-05/en_middle_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1323552198, | |
| "size_mb": 1262.2377376556396, | |
| "size_gb": 1.2326540406793356, | |
| "path": "common_crawl/2020-05/en_middle_0066.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1846094958, | |
| "size_mb": 1760.5733470916748, | |
| "size_gb": 1.7193099092692137, | |
| "path": "common_crawl/2021-04/en_head_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1312749490, | |
| "size_mb": 1251.9354724884033, | |
| "size_gb": 1.2225932348519564, | |
| "path": "common_crawl/2020-05/en_middle_0067.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1425680502, | |
| "size_mb": 1359.6348781585693, | |
| "size_gb": 1.3277684357017279, | |
| "path": "common_crawl/2019-30/en_middle_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00554-of-01024.jsonl", | |
| "size": 845198125, | |
| "size_mb": 806.0437440872192, | |
| "size_gb": 0.787152093835175, | |
| "path": "c4/c4-train.00554-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00028-of-01024.jsonl", | |
| "size": 846004372, | |
| "size_mb": 806.8126411437988, | |
| "size_gb": 0.787902969866991, | |
| "path": "c4/c4-train.00028-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00716-of-01024.jsonl", | |
| "size": 844576776, | |
| "size_mb": 805.4511795043945, | |
| "size_gb": 0.7865734174847603, | |
| "path": "c4/c4-train.00716-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00035-of-01024.jsonl", | |
| "size": 844524269, | |
| "size_mb": 805.401104927063, | |
| "size_gb": 0.786524516530335, | |
| "path": "c4/c4-train.00035-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00121-of-01024.jsonl", | |
| "size": 844628786, | |
| "size_mb": 805.5007801055908, | |
| "size_gb": 0.786621855571866, | |
| "path": "c4/c4-train.00121-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00042-of-01024.jsonl", | |
| "size": 842904345, | |
| "size_mb": 803.8562250137329, | |
| "size_gb": 0.7850158447399735, | |
| "path": "c4/c4-train.00042-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00092-of-01024.jsonl", | |
| "size": 845347894, | |
| "size_mb": 806.1865749359131, | |
| "size_gb": 0.7872915770858526, | |
| "path": "c4/c4-train.00092-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00550-of-01024.jsonl", | |
| "size": 846048446, | |
| "size_mb": 806.8546733856201, | |
| "size_gb": 0.7879440169781446, | |
| "path": "c4/c4-train.00550-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00552-of-01024.jsonl", | |
| "size": 849217548, | |
| "size_mb": 809.8769645690918, | |
| "size_gb": 0.7908954732120037, | |
| "path": "c4/c4-train.00552-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00556-of-01024.jsonl", | |
| "size": 848128624, | |
| "size_mb": 808.8384857177734, | |
| "size_gb": 0.7898813337087631, | |
| "path": "c4/c4-train.00556-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00553-of-01024.jsonl", | |
| "size": 846154150, | |
| "size_mb": 806.9554805755615, | |
| "size_gb": 0.7880424614995718, | |
| "path": "c4/c4-train.00553-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00477-of-01024.jsonl", | |
| "size": 850547624, | |
| "size_mb": 811.1454238891602, | |
| "size_gb": 0.792134203016758, | |
| "path": "c4/c4-train.00477-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_3a5d4f93-97ec-483a-88ef-324df9651b3f.jsonl", | |
| "size": 887007736, | |
| "size_mb": 845.9164962768555, | |
| "size_gb": 0.8260903283953667, | |
| "path": "arxiv/arxiv_3a5d4f93-97ec-483a-88ef-324df9651b3f.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0072.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1426960430, | |
| "size_mb": 1360.8555126190186, | |
| "size_gb": 1.3289604615420103, | |
| "path": "common_crawl/2019-30/en_middle_0072.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2226385651, | |
| "size_mb": 2123.246813774109, | |
| "size_gb": 2.073483216576278, | |
| "path": "common_crawl/2019-30/en_head_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00783-of-01024.jsonl", | |
| "size": 844821925, | |
| "size_mb": 805.6849718093872, | |
| "size_gb": 0.7868017302826047, | |
| "path": "c4/c4-train.00783-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00796-of-01024.jsonl", | |
| "size": 846642643, | |
| "size_mb": 807.4213438034058, | |
| "size_gb": 0.7884974060580134, | |
| "path": "c4/c4-train.00796-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_64e5075e-e87e-4b2a-9e38-e5c102f6f2b1.jsonl", | |
| "size": 842775045, | |
| "size_mb": 803.7329149246216, | |
| "size_gb": 0.7848954247310758, | |
| "path": "arxiv/arxiv_64e5075e-e87e-4b2a-9e38-e5c102f6f2b1.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00326-of-01024.jsonl", | |
| "size": 846255621, | |
| "size_mb": 807.0522508621216, | |
| "size_gb": 0.7881369637325406, | |
| "path": "c4/c4-train.00326-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00795-of-01024.jsonl", | |
| "size": 841844965, | |
| "size_mb": 802.8459215164185, | |
| "size_gb": 0.7840292202308774, | |
| "path": "c4/c4-train.00795-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00310-of-01024.jsonl", | |
| "size": 845236570, | |
| "size_mb": 806.0804080963135, | |
| "size_gb": 0.7871878985315561, | |
| "path": "c4/c4-train.00310-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1842125159, | |
| "size_mb": 1756.7874517440796, | |
| "size_gb": 1.7156127458438277, | |
| "path": "common_crawl/2021-04/en_head_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_8e6bd730-0f10-49d9-9b02-5ce16da47483.jsonl", | |
| "size": 952012287, | |
| "size_mb": 907.9096670150757, | |
| "size_gb": 0.8866305341944098, | |
| "path": "arxiv/arxiv_8e6bd730-0f10-49d9-9b02-5ce16da47483.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00476-of-01024.jsonl", | |
| "size": 844010269, | |
| "size_mb": 804.9109163284302, | |
| "size_gb": 0.7860458167269826, | |
| "path": "c4/c4-train.00476-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_65dd2ff6-dae3-4a60-90d3-c3d7349fc92f.jsonl", | |
| "size": 937049872, | |
| "size_mb": 893.6403961181641, | |
| "size_gb": 0.8726956993341446, | |
| "path": "arxiv/arxiv_65dd2ff6-dae3-4a60-90d3-c3d7349fc92f.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00321-of-01024.jsonl", | |
| "size": 843741697, | |
| "size_mb": 804.6547861099243, | |
| "size_gb": 0.785795689560473, | |
| "path": "c4/c4-train.00321-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00312-of-01024.jsonl", | |
| "size": 844596895, | |
| "size_mb": 805.4703664779663, | |
| "size_gb": 0.786592154763639, | |
| "path": "c4/c4-train.00312-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00316-of-01024.jsonl", | |
| "size": 847754405, | |
| "size_mb": 808.4816026687622, | |
| "size_gb": 0.7895328151062131, | |
| "path": "c4/c4-train.00316-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00480-of-01024.jsonl", | |
| "size": 848117821, | |
| "size_mb": 808.8281831741333, | |
| "size_gb": 0.7898712726309896, | |
| "path": "c4/c4-train.00480-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00475-of-01024.jsonl", | |
| "size": 848800276, | |
| "size_mb": 809.4790229797363, | |
| "size_gb": 0.7905068583786488, | |
| "path": "c4/c4-train.00475-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_393d19f2-1cd1-421f-be8a-78d955fdf602.jsonl", | |
| "size": 1088918487, | |
| "size_mb": 1038.473593711853, | |
| "size_gb": 1.0141343688592315, | |
| "path": "arxiv/arxiv_393d19f2-1cd1-421f-be8a-78d955fdf602.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00479-of-01024.jsonl", | |
| "size": 846883721, | |
| "size_mb": 807.6512537002563, | |
| "size_gb": 0.7887219274416566, | |
| "path": "c4/c4-train.00479-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00314-of-01024.jsonl", | |
| "size": 842431038, | |
| "size_mb": 803.4048442840576, | |
| "size_gb": 0.78457504324615, | |
| "path": "c4/c4-train.00314-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00719-of-01024.jsonl", | |
| "size": 843608667, | |
| "size_mb": 804.5279188156128, | |
| "size_gb": 0.7856717957183719, | |
| "path": "c4/c4-train.00719-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_2f28f1a7-6972-48ad-8997-65a5d52e4f1c.jsonl", | |
| "size": 1035691138, | |
| "size_mb": 987.7120380401611, | |
| "size_gb": 0.9645625371485949, | |
| "path": "arxiv/arxiv_2f28f1a7-6972-48ad-8997-65a5d52e4f1c.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00718-of-01024.jsonl", | |
| "size": 842463855, | |
| "size_mb": 803.4361410140991, | |
| "size_gb": 0.7846056064590812, | |
| "path": "c4/c4-train.00718-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00315-of-01024.jsonl", | |
| "size": 847941104, | |
| "size_mb": 808.6596527099609, | |
| "size_gb": 0.7897066920995712, | |
| "path": "c4/c4-train.00315-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00715-of-01024.jsonl", | |
| "size": 848279548, | |
| "size_mb": 808.9824180603027, | |
| "size_gb": 0.7900218926370144, | |
| "path": "c4/c4-train.00715-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0074.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1351325820, | |
| "size_mb": 1288.7247276306152, | |
| "size_gb": 1.2585202418267727, | |
| "path": "common_crawl/2019-30/en_middle_0074.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00317-of-01024.jsonl", | |
| "size": 846611363, | |
| "size_mb": 807.3915128707886, | |
| "size_gb": 0.7884682742878795, | |
| "path": "c4/c4-train.00317-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00328-of-01024.jsonl", | |
| "size": 842809248, | |
| "size_mb": 803.7655334472656, | |
| "size_gb": 0.7849272787570953, | |
| "path": "c4/c4-train.00328-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00329-of-01024.jsonl", | |
| "size": 845781039, | |
| "size_mb": 806.5996541976929, | |
| "size_gb": 0.7876949748024344, | |
| "path": "c4/c4-train.00329-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00808-of-01024.jsonl", | |
| "size": 848014142, | |
| "size_mb": 808.7293071746826, | |
| "size_gb": 0.789774714037776, | |
| "path": "c4/c4-train.00808-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00478-of-01024.jsonl", | |
| "size": 847758702, | |
| "size_mb": 808.4857006072998, | |
| "size_gb": 0.7895368169993162, | |
| "path": "c4/c4-train.00478-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00691-of-01024.jsonl", | |
| "size": 844096006, | |
| "size_mb": 804.9926815032959, | |
| "size_gb": 0.7861256655305624, | |
| "path": "c4/c4-train.00691-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00809-of-01024.jsonl", | |
| "size": 840160210, | |
| "size_mb": 801.2392139434814, | |
| "size_gb": 0.7824601698666811, | |
| "path": "c4/c4-train.00809-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00454-of-01024.jsonl", | |
| "size": 840754774, | |
| "size_mb": 801.8062343597412, | |
| "size_gb": 0.7830139007419348, | |
| "path": "c4/c4-train.00454-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00807-of-01024.jsonl", | |
| "size": 848273823, | |
| "size_mb": 808.9769582748413, | |
| "size_gb": 0.7900165608152747, | |
| "path": "c4/c4-train.00807-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00452-of-01024.jsonl", | |
| "size": 846279993, | |
| "size_mb": 807.075493812561, | |
| "size_gb": 0.7881596619263291, | |
| "path": "c4/c4-train.00452-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00451-of-01024.jsonl", | |
| "size": 843801472, | |
| "size_mb": 804.7117919921875, | |
| "size_gb": 0.7858513593673706, | |
| "path": "c4/c4-train.00451-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00797-of-01024.jsonl", | |
| "size": 848969546, | |
| "size_mb": 809.6404514312744, | |
| "size_gb": 0.7906645033508539, | |
| "path": "c4/c4-train.00797-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2197576272, | |
| "size_mb": 2095.7720489501953, | |
| "size_gb": 2.046652391552925, | |
| "path": "common_crawl/2019-30/en_head_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00453-of-01024.jsonl", | |
| "size": 846059556, | |
| "size_mb": 806.8652687072754, | |
| "size_gb": 0.7879543639719486, | |
| "path": "c4/c4-train.00453-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00690-of-01024.jsonl", | |
| "size": 850966971, | |
| "size_mb": 811.5453443527222, | |
| "size_gb": 0.7925247503444552, | |
| "path": "c4/c4-train.00690-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_4cc7015c-c39a-4bf6-9686-c00b3343edd9.jsonl", | |
| "size": 940882308, | |
| "size_mb": 897.2952919006348, | |
| "size_gb": 0.8762649334967136, | |
| "path": "arxiv/arxiv_4cc7015c-c39a-4bf6-9686-c00b3343edd9.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1283932035, | |
| "size_mb": 1224.4530057907104, | |
| "size_gb": 1.1957548884674907, | |
| "path": "common_crawl/2020-05/en_middle_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00693-of-01024.jsonl", | |
| "size": 843176391, | |
| "size_mb": 804.115668296814, | |
| "size_gb": 0.7852692073211074, | |
| "path": "c4/c4-train.00693-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0072.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1357110810, | |
| "size_mb": 1294.2417240142822, | |
| "size_gb": 1.2639079336076975, | |
| "path": "common_crawl/2020-05/en_middle_0072.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_5ee33ef7-455e-4fd5-9512-c4771dd802c1.jsonl", | |
| "size": 974732570, | |
| "size_mb": 929.5774173736572, | |
| "size_gb": 0.9077904466539621, | |
| "path": "arxiv/arxiv_5ee33ef7-455e-4fd5-9512-c4771dd802c1.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00694-of-01024.jsonl", | |
| "size": 847075509, | |
| "size_mb": 807.8341569900513, | |
| "size_gb": 0.7889005439355969, | |
| "path": "c4/c4-train.00694-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/arxiv/arxiv_3c89ea11-69ff-4049-b775-f0c785997909.jsonl", | |
| "size": 951841542, | |
| "size_mb": 907.7468318939209, | |
| "size_gb": 0.8864715155214071, | |
| "path": "arxiv/arxiv_3c89ea11-69ff-4049-b775-f0c785997909.jsonl", | |
| "top_folder": "arxiv", | |
| "top_folders": "arxiv" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00695-of-01024.jsonl", | |
| "size": 843353664, | |
| "size_mb": 804.2847290039062, | |
| "size_gb": 0.7854343056678772, | |
| "path": "c4/c4-train.00695-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00455-of-01024.jsonl", | |
| "size": 846005833, | |
| "size_mb": 806.8140344619751, | |
| "size_gb": 0.7879043305292726, | |
| "path": "c4/c4-train.00455-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1277496424, | |
| "size_mb": 1218.315528869629, | |
| "size_gb": 1.189761258661747, | |
| "path": "common_crawl/2020-05/en_middle_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1286358675, | |
| "size_mb": 1226.7672300338745, | |
| "size_gb": 1.1980148730799556, | |
| "path": "common_crawl/2020-05/en_middle_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00327-of-01024.jsonl", | |
| "size": 848126816, | |
| "size_mb": 808.8367614746094, | |
| "size_gb": 0.7898796498775482, | |
| "path": "c4/c4-train.00327-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0073.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1310360688, | |
| "size_mb": 1249.6573333740234, | |
| "size_gb": 1.2203684896230698, | |
| "path": "common_crawl/2020-05/en_middle_0073.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0071.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1309983059, | |
| "size_mb": 1249.2971982955933, | |
| "size_gb": 1.2200167952105403, | |
| "path": "common_crawl/2020-05/en_middle_0071.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1309544322, | |
| "size_mb": 1248.8787860870361, | |
| "size_gb": 1.2196081895381212, | |
| "path": "common_crawl/2020-05/en_middle_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2177597941, | |
| "size_mb": 2076.7192277908325, | |
| "size_gb": 2.028046120889485, | |
| "path": "common_crawl/2019-30/en_head_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1328941422, | |
| "size_mb": 1267.3773021697998, | |
| "size_gb": 1.2376731466501951, | |
| "path": "common_crawl/2020-05/en_middle_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0086.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1586061721, | |
| "size_mb": 1512.5863275527954, | |
| "size_gb": 1.4771350855007768, | |
| "path": "common_crawl/2021-04/en_middle_0086.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2213955590, | |
| "size_mb": 2111.392583847046, | |
| "size_gb": 2.0619068201631308, | |
| "path": "common_crawl/2022-05/en_head_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2031999654, | |
| "size_mb": 1937.8658809661865, | |
| "size_gb": 1.8924471493810415, | |
| "path": "common_crawl/2022-05/en_head_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00458-of-01024.jsonl", | |
| "size": 844106163, | |
| "size_mb": 805.0023679733276, | |
| "size_gb": 0.7861351249739528, | |
| "path": "c4/c4-train.00458-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00698-of-01024.jsonl", | |
| "size": 843720852, | |
| "size_mb": 804.6349067687988, | |
| "size_gb": 0.7857762761414051, | |
| "path": "c4/c4-train.00698-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00696-of-01024.jsonl", | |
| "size": 840130975, | |
| "size_mb": 801.2113332748413, | |
| "size_gb": 0.7824329426512122, | |
| "path": "c4/c4-train.00696-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1278902560, | |
| "size_mb": 1219.6565246582031, | |
| "size_gb": 1.1910708248615265, | |
| "path": "common_crawl/2020-05/en_middle_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00459-of-01024.jsonl", | |
| "size": 845638609, | |
| "size_mb": 806.4638223648071, | |
| "size_gb": 0.787562326528132, | |
| "path": "c4/c4-train.00459-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1300387811, | |
| "size_mb": 1240.1464567184448, | |
| "size_gb": 1.2110805241391063, | |
| "path": "common_crawl/2020-05/en_middle_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2168903469, | |
| "size_mb": 2068.4275331497192, | |
| "size_gb": 2.0199487628415227, | |
| "path": "common_crawl/2019-30/en_head_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00457-of-01024.jsonl", | |
| "size": 843236310, | |
| "size_mb": 804.1728115081787, | |
| "size_gb": 0.7853250112384558, | |
| "path": "c4/c4-train.00457-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00692-of-01024.jsonl", | |
| "size": 850443069, | |
| "size_mb": 811.0457124710083, | |
| "size_gb": 0.792036828584969, | |
| "path": "c4/c4-train.00692-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00699-of-01024.jsonl", | |
| "size": 844284488, | |
| "size_mb": 805.1724319458008, | |
| "size_gb": 0.7863012030720711, | |
| "path": "c4/c4-train.00699-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2245600469, | |
| "size_mb": 2141.5714921951294, | |
| "size_gb": 2.091378410346806, | |
| "path": "common_crawl/2022-05/en_head_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00456-of-01024.jsonl", | |
| "size": 848757084, | |
| "size_mb": 809.4378318786621, | |
| "size_gb": 0.790466632694006, | |
| "path": "c4/c4-train.00456-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00697-of-01024.jsonl", | |
| "size": 845994841, | |
| "size_mb": 806.8035516738892, | |
| "size_gb": 0.7878940934315324, | |
| "path": "c4/c4-train.00697-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1294192581, | |
| "size_mb": 1234.2382249832153, | |
| "size_gb": 1.2053107665851712, | |
| "path": "common_crawl/2020-05/en_middle_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1319184116, | |
| "size_mb": 1258.0720100402832, | |
| "size_gb": 1.228585947304964, | |
| "path": "common_crawl/2020-05/en_middle_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1282166568, | |
| "size_mb": 1222.7693252563477, | |
| "size_gb": 1.194110669195652, | |
| "path": "common_crawl/2020-05/en_middle_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00460-of-01024.jsonl", | |
| "size": 845737937, | |
| "size_mb": 806.5585489273071, | |
| "size_gb": 0.7876548329368234, | |
| "path": "c4/c4-train.00460-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1287797736, | |
| "size_mb": 1228.1396255493164, | |
| "size_gb": 1.1993551030755043, | |
| "path": "common_crawl/2020-05/en_middle_0053.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00563-of-01024.jsonl", | |
| "size": 845588068, | |
| "size_mb": 806.4156227111816, | |
| "size_gb": 0.7875152565538883, | |
| "path": "c4/c4-train.00563-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0086.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1374996066, | |
| "size_mb": 1311.298433303833, | |
| "size_gb": 1.2805648762732744, | |
| "path": "common_crawl/2019-30/en_middle_0086.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00483-of-01024.jsonl", | |
| "size": 844576229, | |
| "size_mb": 805.4506578445435, | |
| "size_gb": 0.786572908051312, | |
| "path": "c4/c4-train.00483-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1346177188, | |
| "size_mb": 1283.814609527588, | |
| "size_gb": 1.253725204616785, | |
| "path": "common_crawl/2020-05/en_middle_0069.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1295904391, | |
| "size_mb": 1235.8707342147827, | |
| "size_gb": 1.2069050138816237, | |
| "path": "common_crawl/2020-05/en_middle_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1249871118, | |
| "size_mb": 1191.9699840545654, | |
| "size_gb": 1.1640331875532866, | |
| "path": "common_crawl/2020-05/en_middle_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2166259607, | |
| "size_mb": 2065.906149864197, | |
| "size_gb": 2.0174864744767547, | |
| "path": "common_crawl/2019-30/en_head_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0091.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1548558340, | |
| "size_mb": 1476.8203163146973, | |
| "size_gb": 1.4422073401510715, | |
| "path": "common_crawl/2021-04/en_middle_0091.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0087.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1438875280, | |
| "size_mb": 1372.2183990478516, | |
| "size_gb": 1.3400570303201675, | |
| "path": "common_crawl/2019-30/en_middle_0087.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00126-of-01024.jsonl", | |
| "size": 842804672, | |
| "size_mb": 803.7611694335938, | |
| "size_gb": 0.7849230170249939, | |
| "path": "c4/c4-train.00126-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2182110467, | |
| "size_mb": 2081.022707939148, | |
| "size_gb": 2.032248738221824, | |
| "path": "common_crawl/2022-05/en_head_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0102.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 64982392, | |
| "size_mb": 61.97203826904297, | |
| "size_gb": 0.060519568622112274, | |
| "path": "common_crawl/2021-04/en_middle_0102.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0088.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1407325738, | |
| "size_mb": 1342.1304111480713, | |
| "size_gb": 1.3106742296367884, | |
| "path": "common_crawl/2019-30/en_middle_0088.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0087.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1572363900, | |
| "size_mb": 1499.5230674743652, | |
| "size_gb": 1.4643779955804348, | |
| "path": "common_crawl/2021-04/en_middle_0087.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2140038158, | |
| "size_mb": 2040.8994274139404, | |
| "size_gb": 1.9930658470839262, | |
| "path": "common_crawl/2022-05/en_head_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0090.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1603391170, | |
| "size_mb": 1529.1129779815674, | |
| "size_gb": 1.4932743925601244, | |
| "path": "common_crawl/2021-04/en_middle_0090.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00125-of-01024.jsonl", | |
| "size": 840496992, | |
| "size_mb": 801.5603942871094, | |
| "size_gb": 0.7827738225460052, | |
| "path": "c4/c4-train.00125-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0088.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1630746281, | |
| "size_mb": 1555.2008447647095, | |
| "size_gb": 1.5187508249655366, | |
| "path": "common_crawl/2021-04/en_middle_0088.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0083.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1613140169, | |
| "size_mb": 1538.4103479385376, | |
| "size_gb": 1.5023538554087281, | |
| "path": "common_crawl/2021-04/en_middle_0083.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0095.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1261717415, | |
| "size_mb": 1203.2674932479858, | |
| "size_gb": 1.1750659113749862, | |
| "path": "common_crawl/2020-05/en_middle_0095.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_head_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2171351636, | |
| "size_mb": 2070.7622871398926, | |
| "size_gb": 2.0222287960350513, | |
| "path": "common_crawl/2019-30/en_head_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00464-of-01024.jsonl", | |
| "size": 842987255, | |
| "size_mb": 803.9352941513062, | |
| "size_gb": 0.7850930606946349, | |
| "path": "c4/c4-train.00464-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0084.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1558144887, | |
| "size_mb": 1485.9627599716187, | |
| "size_gb": 1.4511355077847838, | |
| "path": "common_crawl/2021-04/en_middle_0084.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00462-of-01024.jsonl", | |
| "size": 848586051, | |
| "size_mb": 809.2747220993042, | |
| "size_gb": 0.7903073458001018, | |
| "path": "c4/c4-train.00462-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1327909807, | |
| "size_mb": 1266.3934774398804, | |
| "size_gb": 1.2367123803123832, | |
| "path": "common_crawl/2020-05/en_middle_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00472-of-01024.jsonl", | |
| "size": 846956745, | |
| "size_mb": 807.7208948135376, | |
| "size_gb": 0.7887899363413453, | |
| "path": "c4/c4-train.00472-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00461-of-01024.jsonl", | |
| "size": 849902291, | |
| "size_mb": 810.5299863815308, | |
| "size_gb": 0.7915331898257136, | |
| "path": "c4/c4-train.00461-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00703-of-01024.jsonl", | |
| "size": 845242184, | |
| "size_mb": 806.0857620239258, | |
| "size_gb": 0.78719312697649, | |
| "path": "c4/c4-train.00703-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00710-of-01024.jsonl", | |
| "size": 843362417, | |
| "size_mb": 804.2930765151978, | |
| "size_gb": 0.7854424575343728, | |
| "path": "c4/c4-train.00710-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00474-of-01024.jsonl", | |
| "size": 848659471, | |
| "size_mb": 809.3447408676147, | |
| "size_gb": 0.79037572350353, | |
| "path": "c4/c4-train.00474-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00700-of-01024.jsonl", | |
| "size": 845575235, | |
| "size_mb": 806.4033842086792, | |
| "size_gb": 0.7875033048912883, | |
| "path": "c4/c4-train.00700-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00702-of-01024.jsonl", | |
| "size": 845485346, | |
| "size_mb": 806.3176593780518, | |
| "size_gb": 0.7874195892363787, | |
| "path": "c4/c4-train.00702-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00711-of-01024.jsonl", | |
| "size": 850277970, | |
| "size_mb": 810.888261795044, | |
| "size_gb": 0.7918830681592226, | |
| "path": "c4/c4-train.00711-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00704-of-01024.jsonl", | |
| "size": 848662373, | |
| "size_mb": 809.347508430481, | |
| "size_gb": 0.7903784262016416, | |
| "path": "c4/c4-train.00704-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00701-of-01024.jsonl", | |
| "size": 845176182, | |
| "size_mb": 806.0228176116943, | |
| "size_gb": 0.7871316578239202, | |
| "path": "c4/c4-train.00701-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00712-of-01024.jsonl", | |
| "size": 842255913, | |
| "size_mb": 803.237832069397, | |
| "size_gb": 0.7844119453802705, | |
| "path": "c4/c4-train.00712-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00463-of-01024.jsonl", | |
| "size": 846615525, | |
| "size_mb": 807.3954820632935, | |
| "size_gb": 0.788472150452435, | |
| "path": "c4/c4-train.00463-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00741-of-01024.jsonl", | |
| "size": 845171250, | |
| "size_mb": 806.0181140899658, | |
| "size_gb": 0.7871270645409822, | |
| "path": "c4/c4-train.00741-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00519-of-01024.jsonl", | |
| "size": 847758626, | |
| "size_mb": 808.4856281280518, | |
| "size_gb": 0.7895367462188005, | |
| "path": "c4/c4-train.00519-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00714-of-01024.jsonl", | |
| "size": 849982233, | |
| "size_mb": 810.6062250137329, | |
| "size_gb": 0.7916076416149735, | |
| "path": "c4/c4-train.00714-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00107-of-01024.jsonl", | |
| "size": 848760366, | |
| "size_mb": 809.4409618377686, | |
| "size_gb": 0.7904696892946959, | |
| "path": "c4/c4-train.00107-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00055-of-01024.jsonl", | |
| "size": 844761270, | |
| "size_mb": 805.6271266937256, | |
| "size_gb": 0.7867452409118414, | |
| "path": "c4/c4-train.00055-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00481-of-01024.jsonl", | |
| "size": 847923665, | |
| "size_mb": 808.6430215835571, | |
| "size_gb": 0.7896904507651925, | |
| "path": "c4/c4-train.00481-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00721-of-01024.jsonl", | |
| "size": 845139354, | |
| "size_mb": 805.9876956939697, | |
| "size_gb": 0.7870973590761423, | |
| "path": "c4/c4-train.00721-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00720-of-01024.jsonl", | |
| "size": 843247434, | |
| "size_mb": 804.1834201812744, | |
| "size_gb": 0.7853353712707758, | |
| "path": "c4/c4-train.00720-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00482-of-01024.jsonl", | |
| "size": 845414012, | |
| "size_mb": 806.2496299743652, | |
| "size_gb": 0.787353154271841, | |
| "path": "c4/c4-train.00482-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0089.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1583853946, | |
| "size_mb": 1510.4808292388916, | |
| "size_gb": 1.475078934803605, | |
| "path": "common_crawl/2021-04/en_middle_0089.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00722-of-01024.jsonl", | |
| "size": 844675853, | |
| "size_mb": 805.5456666946411, | |
| "size_gb": 0.7866656901314855, | |
| "path": "c4/c4-train.00722-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00717-of-01024.jsonl", | |
| "size": 840795145, | |
| "size_mb": 801.8447351455688, | |
| "size_gb": 0.7830514991655946, | |
| "path": "c4/c4-train.00717-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00471-of-01024.jsonl", | |
| "size": 845528333, | |
| "size_mb": 806.3586549758911, | |
| "size_gb": 0.7874596239998937, | |
| "path": "c4/c4-train.00471-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0095.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1595237019, | |
| "size_mb": 1521.336573600769, | |
| "size_gb": 1.485680247657001, | |
| "path": "common_crawl/2021-04/en_middle_0095.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00103-of-01024.jsonl", | |
| "size": 847609100, | |
| "size_mb": 808.3430290222168, | |
| "size_gb": 0.7893974892795086, | |
| "path": "c4/c4-train.00103-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0092.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1605088571, | |
| "size_mb": 1530.7317457199097, | |
| "size_gb": 1.4948552204295993, | |
| "path": "common_crawl/2021-04/en_middle_0092.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00016-of-01024.jsonl", | |
| "size": 844901340, | |
| "size_mb": 805.7607078552246, | |
| "size_gb": 0.7868756912648678, | |
| "path": "c4/c4-train.00016-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2228230926, | |
| "size_mb": 2125.0066051483154, | |
| "size_gb": 2.075201762840152, | |
| "path": "common_crawl/2022-05/en_head_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00127-of-01024.jsonl", | |
| "size": 846293776, | |
| "size_mb": 807.0886383056641, | |
| "size_gb": 0.7881724983453751, | |
| "path": "c4/c4-train.00127-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1323174837, | |
| "size_mb": 1261.8778581619263, | |
| "size_gb": 1.2323025958612561, | |
| "path": "common_crawl/2020-05/en_middle_0058.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2177753057, | |
| "size_mb": 2076.867157936096, | |
| "size_gb": 2.028190583921969, | |
| "path": "common_crawl/2022-05/en_head_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1267785764, | |
| "size_mb": 1209.0547218322754, | |
| "size_gb": 1.1807175017893314, | |
| "path": "common_crawl/2020-05/en_middle_0054.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0077.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1277491548, | |
| "size_mb": 1218.310878753662, | |
| "size_gb": 1.1897567175328732, | |
| "path": "common_crawl/2020-05/en_middle_0077.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1324725873, | |
| "size_mb": 1263.3570413589478, | |
| "size_gb": 1.2337471107020974, | |
| "path": "common_crawl/2020-05/en_middle_0068.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00057-of-01024.jsonl", | |
| "size": 847699260, | |
| "size_mb": 808.429012298584, | |
| "size_gb": 0.7894814573228359, | |
| "path": "c4/c4-train.00057-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00530-of-01024.jsonl", | |
| "size": 844266829, | |
| "size_mb": 805.1555910110474, | |
| "size_gb": 0.7862847568467259, | |
| "path": "c4/c4-train.00530-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00723-of-01024.jsonl", | |
| "size": 849436777, | |
| "size_mb": 810.0860376358032, | |
| "size_gb": 0.7910996461287141, | |
| "path": "c4/c4-train.00723-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0112.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1377701619, | |
| "size_mb": 1313.878649711609, | |
| "size_gb": 1.283084618858993, | |
| "path": "common_crawl/2020-05/en_middle_0112.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2203740514, | |
| "size_mb": 2101.650728225708, | |
| "size_gb": 2.052393289282918, | |
| "path": "common_crawl/2022-05/en_head_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1320525530, | |
| "size_mb": 1259.351282119751, | |
| "size_gb": 1.2298352364450693, | |
| "path": "common_crawl/2020-05/en_middle_0065.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1305334013, | |
| "size_mb": 1244.863522529602, | |
| "size_gb": 1.2156870337203145, | |
| "path": "common_crawl/2020-05/en_middle_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0089.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1435410196, | |
| "size_mb": 1368.9138374328613, | |
| "size_gb": 1.3368299193680286, | |
| "path": "common_crawl/2019-30/en_middle_0089.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0085.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1629099938, | |
| "size_mb": 1553.6307697296143, | |
| "size_gb": 1.5172175485640764, | |
| "path": "common_crawl/2021-04/en_middle_0085.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1310669266, | |
| "size_mb": 1249.9516162872314, | |
| "size_gb": 1.2206558752804995, | |
| "path": "common_crawl/2020-05/en_middle_0055.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0124.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1321243437, | |
| "size_mb": 1260.0359315872192, | |
| "size_gb": 1.2305038394406438, | |
| "path": "common_crawl/2020-05/en_middle_0124.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0085.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1292387601, | |
| "size_mb": 1232.5168619155884, | |
| "size_gb": 1.2036297479644418, | |
| "path": "common_crawl/2020-05/en_middle_0085.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00492-of-01024.jsonl", | |
| "size": 848543797, | |
| "size_mb": 809.2344255447388, | |
| "size_gb": 0.790267993696034, | |
| "path": "c4/c4-train.00492-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1269394687, | |
| "size_mb": 1210.5891103744507, | |
| "size_gb": 1.1822159281000495, | |
| "path": "common_crawl/2020-05/en_middle_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1354349125, | |
| "size_mb": 1291.6079759597778, | |
| "size_gb": 1.2613359140232205, | |
| "path": "common_crawl/2020-05/en_middle_0064.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0123.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1346398709, | |
| "size_mb": 1284.0258684158325, | |
| "size_gb": 1.2539315121248364, | |
| "path": "common_crawl/2020-05/en_middle_0123.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2115998738, | |
| "size_mb": 2017.9736499786377, | |
| "size_gb": 1.9706773925572634, | |
| "path": "common_crawl/2022-05/en_head_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00484-of-01024.jsonl", | |
| "size": 846007005, | |
| "size_mb": 806.8151521682739, | |
| "size_gb": 0.78790542203933, | |
| "path": "c4/c4-train.00484-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1287220028, | |
| "size_mb": 1227.588680267334, | |
| "size_gb": 1.1988170705735683, | |
| "path": "common_crawl/2020-05/en_middle_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2074814234, | |
| "size_mb": 1978.6970462799072, | |
| "size_gb": 1.932321334257722, | |
| "path": "common_crawl/2022-05/en_head_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0074.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1319126707, | |
| "size_mb": 1258.0172605514526, | |
| "size_gb": 1.228532481007278, | |
| "path": "common_crawl/2020-05/en_middle_0074.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00727-of-01024.jsonl", | |
| "size": 843670127, | |
| "size_mb": 804.5865316390991, | |
| "size_gb": 0.7857290348038077, | |
| "path": "c4/c4-train.00727-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0082.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1560135318, | |
| "size_mb": 1487.8609828948975, | |
| "size_gb": 1.4529892411082983, | |
| "path": "common_crawl/2021-04/en_middle_0082.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2082838276, | |
| "size_mb": 1986.3493690490723, | |
| "size_gb": 1.9397943057119846, | |
| "path": "common_crawl/2022-05/en_head_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0096.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1575300695, | |
| "size_mb": 1502.3238134384155, | |
| "size_gb": 1.4671130990609527, | |
| "path": "common_crawl/2021-04/en_middle_0096.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2054664514, | |
| "size_mb": 1959.4807758331299, | |
| "size_gb": 1.913555445149541, | |
| "path": "common_crawl/2022-05/en_head_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0079.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1343245932, | |
| "size_mb": 1281.0191459655762, | |
| "size_gb": 1.250995259732008, | |
| "path": "common_crawl/2020-05/en_middle_0079.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00724-of-01024.jsonl", | |
| "size": 842056698, | |
| "size_mb": 803.0478458404541, | |
| "size_gb": 0.7842264119535685, | |
| "path": "c4/c4-train.00724-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0094.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1571062857, | |
| "size_mb": 1498.282296180725, | |
| "size_gb": 1.4631663048639894, | |
| "path": "common_crawl/2021-04/en_middle_0094.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1284635162, | |
| "size_mb": 1225.1235599517822, | |
| "size_gb": 1.1964097265154123, | |
| "path": "common_crawl/2020-05/en_middle_0056.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0081.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1321850098, | |
| "size_mb": 1260.6144886016846, | |
| "size_gb": 1.2310688365250826, | |
| "path": "common_crawl/2020-05/en_middle_0081.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2208050835, | |
| "size_mb": 2105.7613706588745, | |
| "size_gb": 2.056407588534057, | |
| "path": "common_crawl/2022-05/en_head_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0076.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1269054960, | |
| "size_mb": 1210.265121459961, | |
| "size_gb": 1.181899532675743, | |
| "path": "common_crawl/2020-05/en_middle_0076.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2188370766, | |
| "size_mb": 2086.9929943084717, | |
| "size_gb": 2.038079096004367, | |
| "path": "common_crawl/2022-05/en_head_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1359669009, | |
| "size_mb": 1296.6814126968384, | |
| "size_gb": 1.2662904420867562, | |
| "path": "common_crawl/2020-05/en_middle_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1323145072, | |
| "size_mb": 1261.8494720458984, | |
| "size_gb": 1.2322748750448227, | |
| "path": "common_crawl/2020-05/en_middle_0057.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2150061043, | |
| "size_mb": 2050.457995414734, | |
| "size_gb": 2.002400386147201, | |
| "path": "common_crawl/2022-05/en_head_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2222395185, | |
| "size_mb": 2119.4412088394165, | |
| "size_gb": 2.0697668055072427, | |
| "path": "common_crawl/2022-05/en_head_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0075.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1275052853, | |
| "size_mb": 1215.9851579666138, | |
| "size_gb": 1.1874855058267713, | |
| "path": "common_crawl/2020-05/en_middle_0075.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00726-of-01024.jsonl", | |
| "size": 851378369, | |
| "size_mb": 811.9376840591431, | |
| "size_gb": 0.7929078945890069, | |
| "path": "c4/c4-train.00726-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00488-of-01024.jsonl", | |
| "size": 849063777, | |
| "size_mb": 809.7303171157837, | |
| "size_gb": 0.7907522628083825, | |
| "path": "c4/c4-train.00488-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00486-of-01024.jsonl", | |
| "size": 844852420, | |
| "size_mb": 805.714054107666, | |
| "size_gb": 0.7868301309645176, | |
| "path": "c4/c4-train.00486-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00487-of-01024.jsonl", | |
| "size": 845398716, | |
| "size_mb": 806.2350425720215, | |
| "size_gb": 0.7873389087617397, | |
| "path": "c4/c4-train.00487-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00142-of-01024.jsonl", | |
| "size": 844279233, | |
| "size_mb": 805.1674203872681, | |
| "size_gb": 0.7862963089719415, | |
| "path": "c4/c4-train.00142-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00139-of-01024.jsonl", | |
| "size": 849145935, | |
| "size_mb": 809.808669090271, | |
| "size_gb": 0.7908287784084678, | |
| "path": "c4/c4-train.00139-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0078.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1270042140, | |
| "size_mb": 1211.2065696716309, | |
| "size_gb": 1.182818915694952, | |
| "path": "common_crawl/2020-05/en_middle_0078.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00725-of-01024.jsonl", | |
| "size": 847596898, | |
| "size_mb": 808.331392288208, | |
| "size_gb": 0.7893861252814531, | |
| "path": "c4/c4-train.00725-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00516-of-01024.jsonl", | |
| "size": 844219931, | |
| "size_mb": 805.1108655929565, | |
| "size_gb": 0.7862410796806216, | |
| "path": "c4/c4-train.00516-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00140-of-01024.jsonl", | |
| "size": 848558716, | |
| "size_mb": 809.2486534118652, | |
| "size_gb": 0.7902818880975246, | |
| "path": "c4/c4-train.00140-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00023-of-01024.jsonl", | |
| "size": 847759778, | |
| "size_mb": 808.4867267608643, | |
| "size_gb": 0.7895378191024065, | |
| "path": "c4/c4-train.00023-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0080.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1283917097, | |
| "size_mb": 1224.438759803772, | |
| "size_gb": 1.195740976370871, | |
| "path": "common_crawl/2020-05/en_middle_0080.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00485-of-01024.jsonl", | |
| "size": 845082487, | |
| "size_mb": 805.9334630966187, | |
| "size_gb": 0.7870443975552917, | |
| "path": "c4/c4-train.00485-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00145-of-01024.jsonl", | |
| "size": 844322993, | |
| "size_mb": 805.209153175354, | |
| "size_gb": 0.7863370636478066, | |
| "path": "c4/c4-train.00145-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00143-of-01024.jsonl", | |
| "size": 847517092, | |
| "size_mb": 808.2552833557129, | |
| "size_gb": 0.7893118001520634, | |
| "path": "c4/c4-train.00143-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00732-of-01024.jsonl", | |
| "size": 844683452, | |
| "size_mb": 805.5529136657715, | |
| "size_gb": 0.78667276725173, | |
| "path": "c4/c4-train.00732-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00491-of-01024.jsonl", | |
| "size": 845556091, | |
| "size_mb": 806.3851270675659, | |
| "size_gb": 0.7874854756519198, | |
| "path": "c4/c4-train.00491-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00128-of-01024.jsonl", | |
| "size": 843092331, | |
| "size_mb": 804.0355024337769, | |
| "size_gb": 0.7851909203454852, | |
| "path": "c4/c4-train.00128-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00499-of-01024.jsonl", | |
| "size": 842902122, | |
| "size_mb": 803.8541049957275, | |
| "size_gb": 0.7850137744098902, | |
| "path": "c4/c4-train.00499-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00728-of-01024.jsonl", | |
| "size": 844890725, | |
| "size_mb": 805.750584602356, | |
| "size_gb": 0.7868658052757382, | |
| "path": "c4/c4-train.00728-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00137-of-01024.jsonl", | |
| "size": 845334359, | |
| "size_mb": 806.1736669540405, | |
| "size_gb": 0.7872789716348052, | |
| "path": "c4/c4-train.00137-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00135-of-01024.jsonl", | |
| "size": 845419766, | |
| "size_mb": 806.2551174163818, | |
| "size_gb": 0.7873585131019354, | |
| "path": "c4/c4-train.00135-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00739-of-01024.jsonl", | |
| "size": 847537386, | |
| "size_mb": 808.27463722229, | |
| "size_gb": 0.7893307004123926, | |
| "path": "c4/c4-train.00739-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00731-of-01024.jsonl", | |
| "size": 848800228, | |
| "size_mb": 809.4789772033691, | |
| "size_gb": 0.7905068136751652, | |
| "path": "c4/c4-train.00731-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00494-of-01024.jsonl", | |
| "size": 850102215, | |
| "size_mb": 810.720648765564, | |
| "size_gb": 0.7917193835601211, | |
| "path": "c4/c4-train.00494-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00493-of-01024.jsonl", | |
| "size": 842090531, | |
| "size_mb": 803.0801115036011, | |
| "size_gb": 0.7842579213902354, | |
| "path": "c4/c4-train.00493-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00740-of-01024.jsonl", | |
| "size": 845942656, | |
| "size_mb": 806.7537841796875, | |
| "size_gb": 0.7878454923629761, | |
| "path": "c4/c4-train.00740-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00500-of-01024.jsonl", | |
| "size": 844122818, | |
| "size_mb": 805.0182514190674, | |
| "size_gb": 0.786150636151433, | |
| "path": "c4/c4-train.00500-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00495-of-01024.jsonl", | |
| "size": 847567819, | |
| "size_mb": 808.3036603927612, | |
| "size_gb": 0.7893590433523059, | |
| "path": "c4/c4-train.00495-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00146-of-01024.jsonl", | |
| "size": 846729077, | |
| "size_mb": 807.50377368927, | |
| "size_gb": 0.7885779039934278, | |
| "path": "c4/c4-train.00146-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00734-of-01024.jsonl", | |
| "size": 846381671, | |
| "size_mb": 807.1724615097046, | |
| "size_gb": 0.7882543569430709, | |
| "path": "c4/c4-train.00734-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00136-of-01024.jsonl", | |
| "size": 842706352, | |
| "size_mb": 803.6674041748047, | |
| "size_gb": 0.7848314493894577, | |
| "path": "c4/c4-train.00136-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00138-of-01024.jsonl", | |
| "size": 844015526, | |
| "size_mb": 804.9159297943115, | |
| "size_gb": 0.7860507126897573, | |
| "path": "c4/c4-train.00138-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00149-of-01024.jsonl", | |
| "size": 844087172, | |
| "size_mb": 804.9842567443848, | |
| "size_gb": 0.7861174382269382, | |
| "path": "c4/c4-train.00149-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00748-of-01024.jsonl", | |
| "size": 845712507, | |
| "size_mb": 806.5342969894409, | |
| "size_gb": 0.7876311494037509, | |
| "path": "c4/c4-train.00748-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00141-of-01024.jsonl", | |
| "size": 843524216, | |
| "size_mb": 804.447380065918, | |
| "size_gb": 0.785593144595623, | |
| "path": "c4/c4-train.00141-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00134-of-01024.jsonl", | |
| "size": 847645695, | |
| "size_mb": 808.3779287338257, | |
| "size_gb": 0.7894315710291266, | |
| "path": "c4/c4-train.00134-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00733-of-01024.jsonl", | |
| "size": 844838987, | |
| "size_mb": 805.7012434005737, | |
| "size_gb": 0.7868176205083728, | |
| "path": "c4/c4-train.00733-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_middle_0093.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1584207926, | |
| "size_mb": 1510.818410873413, | |
| "size_gb": 1.4754086043685675, | |
| "path": "common_crawl/2021-04/en_middle_0093.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00502-of-01024.jsonl", | |
| "size": 844533589, | |
| "size_mb": 805.4099931716919, | |
| "size_gb": 0.7865331964567304, | |
| "path": "c4/c4-train.00502-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00018-of-01024.jsonl", | |
| "size": 844428392, | |
| "size_mb": 805.3096694946289, | |
| "size_gb": 0.7864352241158485, | |
| "path": "c4/c4-train.00018-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00148-of-01024.jsonl", | |
| "size": 843762451, | |
| "size_mb": 804.674578666687, | |
| "size_gb": 0.7858150182291865, | |
| "path": "c4/c4-train.00148-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0109.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1266517266, | |
| "size_mb": 1207.8449878692627, | |
| "size_gb": 1.1795361209660769, | |
| "path": "common_crawl/2020-05/en_middle_0109.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00501-of-01024.jsonl", | |
| "size": 847009693, | |
| "size_mb": 807.7713899612427, | |
| "size_gb": 0.788839248009026, | |
| "path": "c4/c4-train.00501-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00147-of-01024.jsonl", | |
| "size": 850522222, | |
| "size_mb": 811.1211986541748, | |
| "size_gb": 0.7921105455607176, | |
| "path": "c4/c4-train.00147-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00747-of-01024.jsonl", | |
| "size": 844276327, | |
| "size_mb": 805.1646490097046, | |
| "size_gb": 0.7862936025485396, | |
| "path": "c4/c4-train.00747-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00131-of-01024.jsonl", | |
| "size": 846453331, | |
| "size_mb": 807.2408018112183, | |
| "size_gb": 0.7883210955187678, | |
| "path": "c4/c4-train.00131-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0087.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1282261060, | |
| "size_mb": 1222.8594398498535, | |
| "size_gb": 1.1941986717283726, | |
| "path": "common_crawl/2020-05/en_middle_0087.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00746-of-01024.jsonl", | |
| "size": 848497068, | |
| "size_mb": 809.1898612976074, | |
| "size_gb": 0.7902244739234447, | |
| "path": "c4/c4-train.00746-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00050-of-01024.jsonl", | |
| "size": 842851124, | |
| "size_mb": 803.8054695129395, | |
| "size_gb": 0.7849662788212299, | |
| "path": "c4/c4-train.00050-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2149428841, | |
| "size_mb": 2049.855080604553, | |
| "size_gb": 2.001811602152884, | |
| "path": "common_crawl/2022-05/en_head_0019.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00654-of-01024.jsonl", | |
| "size": 845970032, | |
| "size_mb": 806.7798919677734, | |
| "size_gb": 0.7878709882497787, | |
| "path": "c4/c4-train.00654-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00503-of-01024.jsonl", | |
| "size": 843632407, | |
| "size_mb": 804.5505590438843, | |
| "size_gb": 0.7856939053162932, | |
| "path": "c4/c4-train.00503-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2175145662, | |
| "size_mb": 2074.38055229187, | |
| "size_gb": 2.0257622580975294, | |
| "path": "common_crawl/2022-05/en_head_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0115.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1276099616, | |
| "size_mb": 1216.9834289550781, | |
| "size_gb": 1.1884603798389435, | |
| "path": "common_crawl/2020-05/en_middle_0115.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0088.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1324358334, | |
| "size_mb": 1263.0065288543701, | |
| "size_gb": 1.2334048133343458, | |
| "path": "common_crawl/2020-05/en_middle_0088.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00507-of-01024.jsonl", | |
| "size": 844218056, | |
| "size_mb": 805.1090774536133, | |
| "size_gb": 0.7862393334507942, | |
| "path": "c4/c4-train.00507-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00509-of-01024.jsonl", | |
| "size": 847890158, | |
| "size_mb": 808.6110668182373, | |
| "size_gb": 0.7896592449396849, | |
| "path": "c4/c4-train.00509-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0082.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1316217352, | |
| "size_mb": 1255.2426834106445, | |
| "size_gb": 1.2258229330182076, | |
| "path": "common_crawl/2020-05/en_middle_0082.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00546-of-01024.jsonl", | |
| "size": 847899564, | |
| "size_mb": 808.6200370788574, | |
| "size_gb": 0.7896680049598217, | |
| "path": "c4/c4-train.00546-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00051-of-01024.jsonl", | |
| "size": 844997353, | |
| "size_mb": 805.8522729873657, | |
| "size_gb": 0.7869651103392243, | |
| "path": "c4/c4-train.00051-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00511-of-01024.jsonl", | |
| "size": 845566967, | |
| "size_mb": 806.3954992294312, | |
| "size_gb": 0.7874956047162414, | |
| "path": "c4/c4-train.00511-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00512-of-01024.jsonl", | |
| "size": 844036350, | |
| "size_mb": 804.9357891082764, | |
| "size_gb": 0.7860701065510511, | |
| "path": "c4/c4-train.00512-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00022-of-01024.jsonl", | |
| "size": 844386169, | |
| "size_mb": 805.2694025039673, | |
| "size_gb": 0.7863959008827806, | |
| "path": "c4/c4-train.00022-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0116.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1311489150, | |
| "size_mb": 1250.7335186004639, | |
| "size_gb": 1.2214194517582655, | |
| "path": "common_crawl/2020-05/en_middle_0116.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00510-of-01024.jsonl", | |
| "size": 845056785, | |
| "size_mb": 805.9089517593384, | |
| "size_gb": 0.7870204607024789, | |
| "path": "c4/c4-train.00510-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00749-of-01024.jsonl", | |
| "size": 848523801, | |
| "size_mb": 809.2153558731079, | |
| "size_gb": 0.7902493709698319, | |
| "path": "c4/c4-train.00749-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00506-of-01024.jsonl", | |
| "size": 848178570, | |
| "size_mb": 808.8861179351807, | |
| "size_gb": 0.7899278495460749, | |
| "path": "c4/c4-train.00506-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00102-of-01024.jsonl", | |
| "size": 850996467, | |
| "size_mb": 811.5734739303589, | |
| "size_gb": 0.7925522206351161, | |
| "path": "c4/c4-train.00102-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00514-of-01024.jsonl", | |
| "size": 847232660, | |
| "size_mb": 807.9840278625488, | |
| "size_gb": 0.7890469022095203, | |
| "path": "c4/c4-train.00514-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00505-of-01024.jsonl", | |
| "size": 842263802, | |
| "size_mb": 803.2453556060791, | |
| "size_gb": 0.7844192925840616, | |
| "path": "c4/c4-train.00505-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00045-of-01024.jsonl", | |
| "size": 847291472, | |
| "size_mb": 808.0401153564453, | |
| "size_gb": 0.7891016751527786, | |
| "path": "c4/c4-train.00045-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2186166254, | |
| "size_mb": 2084.8906078338623, | |
| "size_gb": 2.036025984212756, | |
| "path": "common_crawl/2022-05/en_head_0016.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00043-of-01024.jsonl", | |
| "size": 841699974, | |
| "size_mb": 802.7076473236084, | |
| "size_gb": 0.7838941868394613, | |
| "path": "c4/c4-train.00043-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00508-of-01024.jsonl", | |
| "size": 845876788, | |
| "size_mb": 806.6909675598145, | |
| "size_gb": 0.7877841480076313, | |
| "path": "c4/c4-train.00508-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2179456538, | |
| "size_mb": 2078.491724014282, | |
| "size_gb": 2.0297770742326975, | |
| "path": "common_crawl/2022-05/en_head_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0103.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1355987554, | |
| "size_mb": 1293.170503616333, | |
| "size_gb": 1.2628618199378252, | |
| "path": "common_crawl/2020-05/en_middle_0103.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00130-of-01024.jsonl", | |
| "size": 845871958, | |
| "size_mb": 806.6863613128662, | |
| "size_gb": 0.7877796497195959, | |
| "path": "c4/c4-train.00130-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00548-of-01024.jsonl", | |
| "size": 844080272, | |
| "size_mb": 804.9776763916016, | |
| "size_gb": 0.7861110121011734, | |
| "path": "c4/c4-train.00548-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00515-of-01024.jsonl", | |
| "size": 842279964, | |
| "size_mb": 803.2607688903809, | |
| "size_gb": 0.7844343446195126, | |
| "path": "c4/c4-train.00515-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00158-of-01024.jsonl", | |
| "size": 848906130, | |
| "size_mb": 809.5799732208252, | |
| "size_gb": 0.7906054425984621, | |
| "path": "c4/c4-train.00158-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00104-of-01024.jsonl", | |
| "size": 847021271, | |
| "size_mb": 807.782431602478, | |
| "size_gb": 0.788850030861795, | |
| "path": "c4/c4-train.00104-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0099.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1287666985, | |
| "size_mb": 1228.014931678772, | |
| "size_gb": 1.1992333317175508, | |
| "path": "common_crawl/2020-05/en_middle_0099.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00523-of-01024.jsonl", | |
| "size": 850079239, | |
| "size_mb": 810.6987371444702, | |
| "size_gb": 0.7916979854926467, | |
| "path": "c4/c4-train.00523-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2202327291, | |
| "size_mb": 2100.3029737472534, | |
| "size_gb": 2.051077122800052, | |
| "path": "common_crawl/2022-05/en_head_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0094.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1288827624, | |
| "size_mb": 1229.1218032836914, | |
| "size_gb": 1.2003142610192299, | |
| "path": "common_crawl/2020-05/en_middle_0094.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00655-of-01024.jsonl", | |
| "size": 844219130, | |
| "size_mb": 805.1101016998291, | |
| "size_gb": 0.7862403336912394, | |
| "path": "c4/c4-train.00655-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2096419853, | |
| "size_mb": 1999.301770210266, | |
| "size_gb": 1.952443134970963, | |
| "path": "common_crawl/2022-05/en_head_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00658-of-01024.jsonl", | |
| "size": 847828069, | |
| "size_mb": 808.551854133606, | |
| "size_gb": 0.7896014200523496, | |
| "path": "c4/c4-train.00658-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2006785731, | |
| "size_mb": 1913.8200101852417, | |
| "size_gb": 1.868964853696525, | |
| "path": "common_crawl/2022-05/en_head_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0105.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1309848040, | |
| "size_mb": 1249.1684341430664, | |
| "size_gb": 1.2198910489678383, | |
| "path": "common_crawl/2020-05/en_middle_0105.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00595-of-01024.jsonl", | |
| "size": 846063571, | |
| "size_mb": 806.8690977096558, | |
| "size_gb": 0.7879581032320857, | |
| "path": "c4/c4-train.00595-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00418-of-01024.jsonl", | |
| "size": 845976480, | |
| "size_mb": 806.7860412597656, | |
| "size_gb": 0.7878769934177399, | |
| "path": "c4/c4-train.00418-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00195-of-01024.jsonl", | |
| "size": 846213340, | |
| "size_mb": 807.0119285583496, | |
| "size_gb": 0.7880975864827633, | |
| "path": "c4/c4-train.00195-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00606-of-01024.jsonl", | |
| "size": 847133970, | |
| "size_mb": 807.8899097442627, | |
| "size_gb": 0.7889549899846315, | |
| "path": "c4/c4-train.00606-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00415-of-01024.jsonl", | |
| "size": 847332382, | |
| "size_mb": 808.0791301727295, | |
| "size_gb": 0.7891397755593061, | |
| "path": "c4/c4-train.00415-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00513-of-01024.jsonl", | |
| "size": 841288039, | |
| "size_mb": 802.3147954940796, | |
| "size_gb": 0.7835105424746871, | |
| "path": "c4/c4-train.00513-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00427-of-01024.jsonl", | |
| "size": 842368738, | |
| "size_mb": 803.3454303741455, | |
| "size_gb": 0.7845170218497515, | |
| "path": "c4/c4-train.00427-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00034-of-01024.jsonl", | |
| "size": 845290835, | |
| "size_mb": 806.1321592330933, | |
| "size_gb": 0.7872384367510676, | |
| "path": "c4/c4-train.00034-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0104.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1297164049, | |
| "size_mb": 1237.0720376968384, | |
| "size_gb": 1.2080781618133187, | |
| "path": "common_crawl/2020-05/en_middle_0104.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00669-of-01024.jsonl", | |
| "size": 847480121, | |
| "size_mb": 808.220025062561, | |
| "size_gb": 0.7892773682251573, | |
| "path": "c4/c4-train.00669-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00608-of-01024.jsonl", | |
| "size": 847140289, | |
| "size_mb": 807.8959360122681, | |
| "size_gb": 0.7889608750119805, | |
| "path": "c4/c4-train.00608-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00667-of-01024.jsonl", | |
| "size": 847161155, | |
| "size_mb": 807.9158353805542, | |
| "size_gb": 0.7889803079888225, | |
| "path": "c4/c4-train.00667-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0108.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1333055206, | |
| "size_mb": 1271.3005123138428, | |
| "size_gb": 1.241504406556487, | |
| "path": "common_crawl/2020-05/en_middle_0108.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0110.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1268524960, | |
| "size_mb": 1209.7596740722656, | |
| "size_gb": 1.181405931711197, | |
| "path": "common_crawl/2020-05/en_middle_0110.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00020-of-01024.jsonl", | |
| "size": 845473870, | |
| "size_mb": 806.3067150115967, | |
| "size_gb": 0.7874089013785124, | |
| "path": "c4/c4-train.00020-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2056160409, | |
| "size_mb": 1960.9073724746704, | |
| "size_gb": 1.9149486059322953, | |
| "path": "common_crawl/2022-05/en_head_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00044-of-01024.jsonl", | |
| "size": 845009413, | |
| "size_mb": 805.8637742996216, | |
| "size_gb": 0.7869763420894742, | |
| "path": "c4/c4-train.00044-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00610-of-01024.jsonl", | |
| "size": 848617934, | |
| "size_mb": 809.3051280975342, | |
| "size_gb": 0.7903370391577482, | |
| "path": "c4/c4-train.00610-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00071-of-01024.jsonl", | |
| "size": 843679066, | |
| "size_mb": 804.5950565338135, | |
| "size_gb": 0.7857373598963022, | |
| "path": "c4/c4-train.00071-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1912019774, | |
| "size_mb": 1823.4441509246826, | |
| "size_gb": 1.7807071786373854, | |
| "path": "common_crawl/2021-04/en_head_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00060-of-01024.jsonl", | |
| "size": 842167561, | |
| "size_mb": 803.1535730361938, | |
| "size_gb": 0.784329661168158, | |
| "path": "c4/c4-train.00060-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00520-of-01024.jsonl", | |
| "size": 842839520, | |
| "size_mb": 803.7944030761719, | |
| "size_gb": 0.7849554717540741, | |
| "path": "c4/c4-train.00520-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2246056552, | |
| "size_mb": 2142.006446838379, | |
| "size_gb": 2.0918031707406044, | |
| "path": "common_crawl/2022-05/en_head_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00161-of-01024.jsonl", | |
| "size": 844305743, | |
| "size_mb": 805.192702293396, | |
| "size_gb": 0.7863209983333945, | |
| "path": "c4/c4-train.00161-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0097.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1380117806, | |
| "size_mb": 1316.1829051971436, | |
| "size_gb": 1.2853348683565855, | |
| "path": "common_crawl/2020-05/en_middle_0097.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2002963700, | |
| "size_mb": 1910.1750373840332, | |
| "size_gb": 1.865405309945345, | |
| "path": "common_crawl/2020-05/en_head_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00429-of-01024.jsonl", | |
| "size": 844244317, | |
| "size_mb": 805.1341218948364, | |
| "size_gb": 0.7862637909129262, | |
| "path": "c4/c4-train.00429-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00435-of-01024.jsonl", | |
| "size": 846428347, | |
| "size_mb": 807.2169752120972, | |
| "size_gb": 0.7882978273555636, | |
| "path": "c4/c4-train.00435-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0106.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1292068872, | |
| "size_mb": 1232.2128982543945, | |
| "size_gb": 1.2033329084515572, | |
| "path": "common_crawl/2020-05/en_middle_0106.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1967675132, | |
| "size_mb": 1876.5212364196777, | |
| "size_gb": 1.8325402699410915, | |
| "path": "common_crawl/2022-05/en_head_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00745-of-01024.jsonl", | |
| "size": 844366870, | |
| "size_mb": 805.250997543335, | |
| "size_gb": 0.786377927288413, | |
| "path": "c4/c4-train.00745-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0101.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1364939990, | |
| "size_mb": 1301.7082118988037, | |
| "size_gb": 1.2711994256824255, | |
| "path": "common_crawl/2020-05/en_middle_0101.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00077-of-01024.jsonl", | |
| "size": 845620609, | |
| "size_mb": 806.4466562271118, | |
| "size_gb": 0.7875455627217889, | |
| "path": "c4/c4-train.00077-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00547-of-01024.jsonl", | |
| "size": 846912407, | |
| "size_mb": 807.6786108016968, | |
| "size_gb": 0.788748643361032, | |
| "path": "c4/c4-train.00547-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00030-of-01024.jsonl", | |
| "size": 844266735, | |
| "size_mb": 805.1555013656616, | |
| "size_gb": 0.7862846693024039, | |
| "path": "c4/c4-train.00030-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00017-of-01024.jsonl", | |
| "size": 848938682, | |
| "size_mb": 809.6110172271729, | |
| "size_gb": 0.790635759010911, | |
| "path": "c4/c4-train.00017-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00549-of-01024.jsonl", | |
| "size": 840887464, | |
| "size_mb": 801.9327774047852, | |
| "size_gb": 0.7831374779343605, | |
| "path": "c4/c4-train.00549-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2226779574, | |
| "size_mb": 2123.6224880218506, | |
| "size_gb": 2.0738500859588385, | |
| "path": "common_crawl/2022-05/en_head_0017.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0093.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1274236899, | |
| "size_mb": 1215.2070035934448, | |
| "size_gb": 1.1867255894467235, | |
| "path": "common_crawl/2020-05/en_middle_0093.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00208-of-01024.jsonl", | |
| "size": 848006602, | |
| "size_mb": 808.7221164703369, | |
| "size_gb": 0.7897676918655634, | |
| "path": "c4/c4-train.00208-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00518-of-01024.jsonl", | |
| "size": 844505975, | |
| "size_mb": 805.3836584091187, | |
| "size_gb": 0.7865074789151549, | |
| "path": "c4/c4-train.00518-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00521-of-01024.jsonl", | |
| "size": 842649088, | |
| "size_mb": 803.61279296875, | |
| "size_gb": 0.7847781181335449, | |
| "path": "c4/c4-train.00521-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00100-of-01024.jsonl", | |
| "size": 847433158, | |
| "size_mb": 808.1752376556396, | |
| "size_gb": 0.7892336305230856, | |
| "path": "c4/c4-train.00100-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00026-of-01024.jsonl", | |
| "size": 843317052, | |
| "size_mb": 804.249813079834, | |
| "size_gb": 0.7854002080857754, | |
| "path": "c4/c4-train.00026-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00084-of-01024.jsonl", | |
| "size": 846473890, | |
| "size_mb": 807.2604084014893, | |
| "size_gb": 0.7883402425795794, | |
| "path": "c4/c4-train.00084-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0086.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1282393139, | |
| "size_mb": 1222.9854001998901, | |
| "size_gb": 1.1943216798827052, | |
| "path": "common_crawl/2020-05/en_middle_0086.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00036-of-01024.jsonl", | |
| "size": 844830741, | |
| "size_mb": 805.6933794021606, | |
| "size_gb": 0.7868099408224225, | |
| "path": "c4/c4-train.00036-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2173646877, | |
| "size_mb": 2072.951199531555, | |
| "size_gb": 2.0243664057925344, | |
| "path": "common_crawl/2022-05/en_head_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00120-of-01024.jsonl", | |
| "size": 848179715, | |
| "size_mb": 808.887209892273, | |
| "size_gb": 0.7899289159104228, | |
| "path": "c4/c4-train.00120-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00066-of-01024.jsonl", | |
| "size": 850120457, | |
| "size_mb": 810.7380456924438, | |
| "size_gb": 0.7917363727465272, | |
| "path": "c4/c4-train.00066-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00074-of-01024.jsonl", | |
| "size": 846688326, | |
| "size_mb": 807.4649105072021, | |
| "size_gb": 0.7885399516671896, | |
| "path": "c4/c4-train.00074-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00517-of-01024.jsonl", | |
| "size": 848227897, | |
| "size_mb": 808.933159828186, | |
| "size_gb": 0.7899737888947129, | |
| "path": "c4/c4-train.00517-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00090-of-01024.jsonl", | |
| "size": 844105549, | |
| "size_mb": 805.0017824172974, | |
| "size_gb": 0.786134553141892, | |
| "path": "c4/c4-train.00090-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00054-of-01024.jsonl", | |
| "size": 849708543, | |
| "size_mb": 810.3452138900757, | |
| "size_gb": 0.791352747939527, | |
| "path": "c4/c4-train.00054-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00058-of-01024.jsonl", | |
| "size": 844526899, | |
| "size_mb": 805.4036130905151, | |
| "size_gb": 0.7865269659087062, | |
| "path": "c4/c4-train.00058-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00541-of-01024.jsonl", | |
| "size": 843076663, | |
| "size_mb": 804.0205602645874, | |
| "size_gb": 0.7851763283833861, | |
| "path": "c4/c4-train.00541-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00543-of-01024.jsonl", | |
| "size": 849933200, | |
| "size_mb": 810.5594635009766, | |
| "size_gb": 0.7915619760751724, | |
| "path": "c4/c4-train.00543-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00598-of-01024.jsonl", | |
| "size": 845487427, | |
| "size_mb": 806.3196439743042, | |
| "size_gb": 0.7874215273186564, | |
| "path": "c4/c4-train.00598-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00109-of-01024.jsonl", | |
| "size": 842320266, | |
| "size_mb": 803.2992038726807, | |
| "size_gb": 0.7844718787819147, | |
| "path": "c4/c4-train.00109-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0111.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1321263004, | |
| "size_mb": 1260.0545921325684, | |
| "size_gb": 1.2305220626294613, | |
| "path": "common_crawl/2020-05/en_middle_0111.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0114.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1276392721, | |
| "size_mb": 1217.2629556655884, | |
| "size_gb": 1.1887333551421762, | |
| "path": "common_crawl/2020-05/en_middle_0114.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1821539834, | |
| "size_mb": 1737.155755996704, | |
| "size_gb": 1.6964411679655313, | |
| "path": "common_crawl/2021-04/en_head_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00047-of-01024.jsonl", | |
| "size": 842921926, | |
| "size_mb": 803.8729915618896, | |
| "size_gb": 0.7850322183221579, | |
| "path": "c4/c4-train.00047-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2231067065, | |
| "size_mb": 2127.7113580703735, | |
| "size_gb": 2.077843123115599, | |
| "path": "common_crawl/2022-05/en_head_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00542-of-01024.jsonl", | |
| "size": 844548223, | |
| "size_mb": 805.4239492416382, | |
| "size_gb": 0.7865468254312873, | |
| "path": "c4/c4-train.00542-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2013940547, | |
| "size_mb": 1920.6433744430542, | |
| "size_gb": 1.8756282953545451, | |
| "path": "common_crawl/2020-05/en_head_0020.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1816392931, | |
| "size_mb": 1732.2472867965698, | |
| "size_gb": 1.6916477410122752, | |
| "path": "common_crawl/2021-04/en_head_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2197402202, | |
| "size_mb": 2095.6060428619385, | |
| "size_gb": 2.046490276232362, | |
| "path": "common_crawl/2022-05/en_head_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1869485507, | |
| "size_mb": 1782.8803129196167, | |
| "size_gb": 1.7410940555855632, | |
| "path": "common_crawl/2021-04/en_head_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0107.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1327292669, | |
| "size_mb": 1265.804928779602, | |
| "size_gb": 1.2361376257613301, | |
| "path": "common_crawl/2020-05/en_middle_0107.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2059666243, | |
| "size_mb": 1964.2507963180542, | |
| "size_gb": 1.9182136682793498, | |
| "path": "common_crawl/2020-05/en_head_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2022-05/en_head_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2188301448, | |
| "size_mb": 2086.926887512207, | |
| "size_gb": 2.0380145385861397, | |
| "path": "common_crawl/2022-05/en_head_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2022-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00029-of-01024.jsonl", | |
| "size": 845001988, | |
| "size_mb": 805.8566932678223, | |
| "size_gb": 0.7869694270193577, | |
| "path": "c4/c4-train.00029-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00082-of-01024.jsonl", | |
| "size": 842155426, | |
| "size_mb": 803.1420001983643, | |
| "size_gb": 0.7843183595687151, | |
| "path": "c4/c4-train.00082-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0100.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1273450998, | |
| "size_mb": 1214.4575099945068, | |
| "size_gb": 1.1859936621040106, | |
| "path": "common_crawl/2020-05/en_middle_0100.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00675-of-01024.jsonl", | |
| "size": 843354700, | |
| "size_mb": 804.285717010498, | |
| "size_gb": 0.7854352705180645, | |
| "path": "c4/c4-train.00675-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00039-of-01024.jsonl", | |
| "size": 848379080, | |
| "size_mb": 809.0773391723633, | |
| "size_gb": 0.790114589035511, | |
| "path": "c4/c4-train.00039-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00206-of-01024.jsonl", | |
| "size": 849422080, | |
| "size_mb": 810.072021484375, | |
| "size_gb": 0.791085958480835, | |
| "path": "c4/c4-train.00206-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1979899578, | |
| "size_mb": 1888.1793766021729, | |
| "size_gb": 1.8439251724630594, | |
| "path": "common_crawl/2020-05/en_head_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1821882448, | |
| "size_mb": 1737.4824981689453, | |
| "size_gb": 1.6967602521181107, | |
| "path": "common_crawl/2021-04/en_head_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00081-of-01024.jsonl", | |
| "size": 842792912, | |
| "size_mb": 803.7499542236328, | |
| "size_gb": 0.7849120646715164, | |
| "path": "c4/c4-train.00081-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00040-of-01024.jsonl", | |
| "size": 848831561, | |
| "size_mb": 809.5088586807251, | |
| "size_gb": 0.7905359948053956, | |
| "path": "c4/c4-train.00040-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00545-of-01024.jsonl", | |
| "size": 842460201, | |
| "size_mb": 803.432656288147, | |
| "size_gb": 0.7846022034063935, | |
| "path": "c4/c4-train.00545-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2067733918, | |
| "size_mb": 1971.944730758667, | |
| "size_gb": 1.9257272761315107, | |
| "path": "common_crawl/2020-05/en_head_0049.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00087-of-01024.jsonl", | |
| "size": 846428656, | |
| "size_mb": 807.2172698974609, | |
| "size_gb": 0.7882981151342392, | |
| "path": "c4/c4-train.00087-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00524-of-01024.jsonl", | |
| "size": 843167357, | |
| "size_mb": 804.1070528030396, | |
| "size_gb": 0.7852607937529683, | |
| "path": "c4/c4-train.00524-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00890-of-01024.jsonl", | |
| "size": 847687223, | |
| "size_mb": 808.4175329208374, | |
| "size_gb": 0.7894702469930053, | |
| "path": "c4/c4-train.00890-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00086-of-01024.jsonl", | |
| "size": 846417484, | |
| "size_mb": 807.206615447998, | |
| "size_gb": 0.7882877103984356, | |
| "path": "c4/c4-train.00086-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00031-of-01024.jsonl", | |
| "size": 845651343, | |
| "size_mb": 806.4759664535522, | |
| "size_gb": 0.7875741859897971, | |
| "path": "c4/c4-train.00031-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00079-of-01024.jsonl", | |
| "size": 850837152, | |
| "size_mb": 811.4215393066406, | |
| "size_gb": 0.7924038469791412, | |
| "path": "c4/c4-train.00079-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00088-of-01024.jsonl", | |
| "size": 844780245, | |
| "size_mb": 805.6452226638794, | |
| "size_gb": 0.7867629127576947, | |
| "path": "c4/c4-train.00088-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00025-of-01024.jsonl", | |
| "size": 845752699, | |
| "size_mb": 806.5726270675659, | |
| "size_gb": 0.7876685811206698, | |
| "path": "c4/c4-train.00025-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00526-of-01024.jsonl", | |
| "size": 847198474, | |
| "size_mb": 807.9514255523682, | |
| "size_gb": 0.7890150640159845, | |
| "path": "c4/c4-train.00526-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00539-of-01024.jsonl", | |
| "size": 849710961, | |
| "size_mb": 810.3475198745728, | |
| "size_gb": 0.7913549998775125, | |
| "path": "c4/c4-train.00539-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2084138696, | |
| "size_mb": 1987.5895462036133, | |
| "size_gb": 1.941005416214466, | |
| "path": "common_crawl/2020-05/en_head_0038.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00544-of-01024.jsonl", | |
| "size": 845496400, | |
| "size_mb": 806.3282012939453, | |
| "size_gb": 0.7874298840761185, | |
| "path": "c4/c4-train.00544-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00078-of-01024.jsonl", | |
| "size": 849027418, | |
| "size_mb": 809.6956424713135, | |
| "size_gb": 0.7907184008508921, | |
| "path": "c4/c4-train.00078-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00046-of-01024.jsonl", | |
| "size": 845150384, | |
| "size_mb": 805.9982147216797, | |
| "size_gb": 0.7871076315641403, | |
| "path": "c4/c4-train.00046-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00529-of-01024.jsonl", | |
| "size": 843760376, | |
| "size_mb": 804.6725997924805, | |
| "size_gb": 0.7858130857348442, | |
| "path": "c4/c4-train.00529-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00560-of-01024.jsonl", | |
| "size": 843187909, | |
| "size_mb": 804.1266527175903, | |
| "size_gb": 0.7852799342945218, | |
| "path": "c4/c4-train.00560-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2054920541, | |
| "size_mb": 1959.7249422073364, | |
| "size_gb": 1.913793888874352, | |
| "path": "common_crawl/2020-05/en_head_0037.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00076-of-01024.jsonl", | |
| "size": 848620437, | |
| "size_mb": 809.3075151443481, | |
| "size_gb": 0.7903393702581525, | |
| "path": "c4/c4-train.00076-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00525-of-01024.jsonl", | |
| "size": 846817790, | |
| "size_mb": 807.5883769989014, | |
| "size_gb": 0.7886605244129896, | |
| "path": "c4/c4-train.00525-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00540-of-01024.jsonl", | |
| "size": 848357944, | |
| "size_mb": 809.0571823120117, | |
| "size_gb": 0.7900949046015739, | |
| "path": "c4/c4-train.00540-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0113.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1253890541, | |
| "size_mb": 1195.803204536438, | |
| "size_gb": 1.1677765669301152, | |
| "path": "common_crawl/2020-05/en_middle_0113.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00027-of-01024.jsonl", | |
| "size": 849384617, | |
| "size_mb": 810.0362939834595, | |
| "size_gb": 0.7910510683432221, | |
| "path": "c4/c4-train.00027-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00870-of-01024.jsonl", | |
| "size": 843790742, | |
| "size_mb": 804.7015590667725, | |
| "size_gb": 0.785841366276145, | |
| "path": "c4/c4-train.00870-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00873-of-01024.jsonl", | |
| "size": 844464624, | |
| "size_mb": 805.3442230224609, | |
| "size_gb": 0.786468967795372, | |
| "path": "c4/c4-train.00873-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00098-of-01024.jsonl", | |
| "size": 851484897, | |
| "size_mb": 812.0392770767212, | |
| "size_gb": 0.7930071065202355, | |
| "path": "c4/c4-train.00098-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00063-of-01024.jsonl", | |
| "size": 844556640, | |
| "size_mb": 805.4319763183594, | |
| "size_gb": 0.7865546643733978, | |
| "path": "c4/c4-train.00063-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00872-of-01024.jsonl", | |
| "size": 847230357, | |
| "size_mb": 807.9818315505981, | |
| "size_gb": 0.789044757373631, | |
| "path": "c4/c4-train.00872-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00946-of-01024.jsonl", | |
| "size": 854378098, | |
| "size_mb": 814.7984485626221, | |
| "size_gb": 0.7957016099244356, | |
| "path": "c4/c4-train.00946-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00091-of-01024.jsonl", | |
| "size": 847330540, | |
| "size_mb": 808.0773735046387, | |
| "size_gb": 0.7891380600631237, | |
| "path": "c4/c4-train.00091-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00889-of-01024.jsonl", | |
| "size": 846118274, | |
| "size_mb": 806.9212665557861, | |
| "size_gb": 0.7880090493708849, | |
| "path": "c4/c4-train.00889-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1866289479, | |
| "size_mb": 1779.8323431015015, | |
| "size_gb": 1.73811752256006, | |
| "path": "common_crawl/2021-04/en_head_0005.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00067-of-01024.jsonl", | |
| "size": 846149795, | |
| "size_mb": 806.9513273239136, | |
| "size_gb": 0.7880384055897593, | |
| "path": "c4/c4-train.00067-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00568-of-01024.jsonl", | |
| "size": 844358804, | |
| "size_mb": 805.2433052062988, | |
| "size_gb": 0.7863704152405262, | |
| "path": "c4/c4-train.00568-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00871-of-01024.jsonl", | |
| "size": 848624620, | |
| "size_mb": 809.3115043640137, | |
| "size_gb": 0.7903432659804821, | |
| "path": "c4/c4-train.00871-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00097-of-01024.jsonl", | |
| "size": 847550954, | |
| "size_mb": 808.287576675415, | |
| "size_gb": 0.789343336597085, | |
| "path": "c4/c4-train.00097-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00024-of-01024.jsonl", | |
| "size": 842566872, | |
| "size_mb": 803.5343856811523, | |
| "size_gb": 0.7847015485167503, | |
| "path": "c4/c4-train.00024-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00933-of-01024.jsonl", | |
| "size": 849148384, | |
| "size_mb": 809.8110046386719, | |
| "size_gb": 0.790831059217453, | |
| "path": "c4/c4-train.00933-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00106-of-01024.jsonl", | |
| "size": 845248984, | |
| "size_mb": 806.0922470092773, | |
| "size_gb": 0.7871994599699974, | |
| "path": "c4/c4-train.00106-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00948-of-01024.jsonl", | |
| "size": 849068221, | |
| "size_mb": 809.7345552444458, | |
| "size_gb": 0.7907564016059041, | |
| "path": "c4/c4-train.00948-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00930-of-01024.jsonl", | |
| "size": 845888459, | |
| "size_mb": 806.7020978927612, | |
| "size_gb": 0.7877950174733996, | |
| "path": "c4/c4-train.00930-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00089-of-01024.jsonl", | |
| "size": 846061995, | |
| "size_mb": 806.8675947189331, | |
| "size_gb": 0.7879566354677081, | |
| "path": "c4/c4-train.00089-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00943-of-01024.jsonl", | |
| "size": 844960565, | |
| "size_mb": 805.8171892166138, | |
| "size_gb": 0.7869308488443494, | |
| "path": "c4/c4-train.00943-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00122-of-01024.jsonl", | |
| "size": 850476293, | |
| "size_mb": 811.0773973464966, | |
| "size_gb": 0.7920677708461881, | |
| "path": "c4/c4-train.00122-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00567-of-01024.jsonl", | |
| "size": 847229816, | |
| "size_mb": 807.981315612793, | |
| "size_gb": 0.7890442535281181, | |
| "path": "c4/c4-train.00567-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00949-of-01024.jsonl", | |
| "size": 843008586, | |
| "size_mb": 803.9556369781494, | |
| "size_gb": 0.785112926736474, | |
| "path": "c4/c4-train.00949-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00068-of-01024.jsonl", | |
| "size": 845245446, | |
| "size_mb": 806.0888729095459, | |
| "size_gb": 0.7871961649507284, | |
| "path": "c4/c4-train.00068-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00110-of-01024.jsonl", | |
| "size": 847348496, | |
| "size_mb": 808.0944976806641, | |
| "size_gb": 0.7891547828912735, | |
| "path": "c4/c4-train.00110-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00096-of-01024.jsonl", | |
| "size": 848886432, | |
| "size_mb": 809.5611877441406, | |
| "size_gb": 0.7905870974063873, | |
| "path": "c4/c4-train.00096-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1807247338, | |
| "size_mb": 1723.525369644165, | |
| "size_gb": 1.68313024379313, | |
| "path": "common_crawl/2021-04/en_head_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00932-of-01024.jsonl", | |
| "size": 844546653, | |
| "size_mb": 805.4224519729614, | |
| "size_gb": 0.7865453632548451, | |
| "path": "c4/c4-train.00932-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00075-of-01024.jsonl", | |
| "size": 849466025, | |
| "size_mb": 810.1139307022095, | |
| "size_gb": 0.7911268854513764, | |
| "path": "c4/c4-train.00075-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00961-of-01024.jsonl", | |
| "size": 844172888, | |
| "size_mb": 805.0660018920898, | |
| "size_gb": 0.786197267472744, | |
| "path": "c4/c4-train.00961-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00947-of-01024.jsonl", | |
| "size": 845758532, | |
| "size_mb": 806.5781898498535, | |
| "size_gb": 0.7876740135252476, | |
| "path": "c4/c4-train.00947-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00070-of-01024.jsonl", | |
| "size": 845671837, | |
| "size_mb": 806.4955110549927, | |
| "size_gb": 0.7875932725146413, | |
| "path": "c4/c4-train.00070-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00117-of-01024.jsonl", | |
| "size": 843304461, | |
| "size_mb": 804.2378053665161, | |
| "size_gb": 0.7853884818032384, | |
| "path": "c4/c4-train.00117-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00566-of-01024.jsonl", | |
| "size": 844290667, | |
| "size_mb": 805.1783246994019, | |
| "size_gb": 0.7863069577142596, | |
| "path": "c4/c4-train.00566-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00564-of-01024.jsonl", | |
| "size": 846809888, | |
| "size_mb": 807.5808410644531, | |
| "size_gb": 0.788653165102005, | |
| "path": "c4/c4-train.00564-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00953-of-01024.jsonl", | |
| "size": 844349422, | |
| "size_mb": 805.2343578338623, | |
| "size_gb": 0.7863616775721312, | |
| "path": "c4/c4-train.00953-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00095-of-01024.jsonl", | |
| "size": 847378968, | |
| "size_mb": 808.1235580444336, | |
| "size_gb": 0.7891831621527672, | |
| "path": "c4/c4-train.00095-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00944-of-01024.jsonl", | |
| "size": 847629929, | |
| "size_mb": 808.3628931045532, | |
| "size_gb": 0.7894168877974153, | |
| "path": "c4/c4-train.00944-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00569-of-01024.jsonl", | |
| "size": 848139027, | |
| "size_mb": 808.848406791687, | |
| "size_gb": 0.7898910222575068, | |
| "path": "c4/c4-train.00569-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00931-of-01024.jsonl", | |
| "size": 845858125, | |
| "size_mb": 806.6731691360474, | |
| "size_gb": 0.7877667667344213, | |
| "path": "c4/c4-train.00931-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1865034724, | |
| "size_mb": 1778.6357154846191, | |
| "size_gb": 1.7369489409029484, | |
| "path": "common_crawl/2021-04/en_head_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00945-of-01024.jsonl", | |
| "size": 845449748, | |
| "size_mb": 806.2837104797363, | |
| "size_gb": 0.7873864360153675, | |
| "path": "c4/c4-train.00945-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00897-of-01024.jsonl", | |
| "size": 849589304, | |
| "size_mb": 810.2314987182617, | |
| "size_gb": 0.7912416979670525, | |
| "path": "c4/c4-train.00897-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0120.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1279088202, | |
| "size_mb": 1219.8335666656494, | |
| "size_gb": 1.1912437174469233, | |
| "path": "common_crawl/2020-05/en_middle_0120.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00893-of-01024.jsonl", | |
| "size": 847307681, | |
| "size_mb": 808.0555734634399, | |
| "size_gb": 0.7891167709603906, | |
| "path": "c4/c4-train.00893-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1874779671, | |
| "size_mb": 1787.9292211532593, | |
| "size_gb": 1.7460246300324798, | |
| "path": "common_crawl/2021-04/en_head_0006.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00021-of-01024.jsonl", | |
| "size": 844618371, | |
| "size_mb": 805.4908475875854, | |
| "size_gb": 0.7866121558472514, | |
| "path": "c4/c4-train.00021-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00094-of-01024.jsonl", | |
| "size": 848914428, | |
| "size_mb": 809.5878868103027, | |
| "size_gb": 0.7906131707131863, | |
| "path": "c4/c4-train.00094-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00069-of-01024.jsonl", | |
| "size": 846395817, | |
| "size_mb": 807.1859521865845, | |
| "size_gb": 0.7882675314322114, | |
| "path": "c4/c4-train.00069-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00894-of-01024.jsonl", | |
| "size": 846985738, | |
| "size_mb": 807.7485446929932, | |
| "size_gb": 0.7888169381767511, | |
| "path": "c4/c4-train.00894-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00896-of-01024.jsonl", | |
| "size": 848477960, | |
| "size_mb": 809.1716384887695, | |
| "size_gb": 0.790206678211689, | |
| "path": "c4/c4-train.00896-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00955-of-01024.jsonl", | |
| "size": 847792285, | |
| "size_mb": 808.5177278518677, | |
| "size_gb": 0.7895680936053395, | |
| "path": "c4/c4-train.00955-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00954-of-01024.jsonl", | |
| "size": 846162284, | |
| "size_mb": 806.9632377624512, | |
| "size_gb": 0.7880500368773937, | |
| "path": "c4/c4-train.00954-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00895-of-01024.jsonl", | |
| "size": 841113993, | |
| "size_mb": 802.1488122940063, | |
| "size_gb": 0.7833484495058656, | |
| "path": "c4/c4-train.00895-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00902-of-01024.jsonl", | |
| "size": 849164690, | |
| "size_mb": 809.8265552520752, | |
| "size_gb": 0.7908462453633547, | |
| "path": "c4/c4-train.00902-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00907-of-01024.jsonl", | |
| "size": 845001243, | |
| "size_mb": 805.8559827804565, | |
| "size_gb": 0.7869687331840396, | |
| "path": "c4/c4-train.00907-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00904-of-01024.jsonl", | |
| "size": 847403396, | |
| "size_mb": 808.1468544006348, | |
| "size_gb": 0.7892059125006199, | |
| "path": "c4/c4-train.00904-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1852622639, | |
| "size_mb": 1766.7986288070679, | |
| "size_gb": 1.7253892859444022, | |
| "path": "common_crawl/2021-04/en_head_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00956-of-01024.jsonl", | |
| "size": 846903048, | |
| "size_mb": 807.6696853637695, | |
| "size_gb": 0.7887399271130562, | |
| "path": "c4/c4-train.00956-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00561-of-01024.jsonl", | |
| "size": 847529359, | |
| "size_mb": 808.2669820785522, | |
| "size_gb": 0.7893232246860862, | |
| "path": "c4/c4-train.00561-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1850748682, | |
| "size_mb": 1765.0114841461182, | |
| "size_gb": 1.7236440274864435, | |
| "path": "common_crawl/2021-04/en_head_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00898-of-01024.jsonl", | |
| "size": 846488020, | |
| "size_mb": 807.2738838195801, | |
| "size_gb": 0.7883534021675587, | |
| "path": "c4/c4-train.00898-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00062-of-01024.jsonl", | |
| "size": 844973112, | |
| "size_mb": 805.8291549682617, | |
| "size_gb": 0.7869425341486931, | |
| "path": "c4/c4-train.00062-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1859825670, | |
| "size_mb": 1773.667974472046, | |
| "size_gb": 1.7320976313203573, | |
| "path": "common_crawl/2021-04/en_head_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00565-of-01024.jsonl", | |
| "size": 844506571, | |
| "size_mb": 805.3842267990112, | |
| "size_gb": 0.7865080339834094, | |
| "path": "c4/c4-train.00565-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00108-of-01024.jsonl", | |
| "size": 848054095, | |
| "size_mb": 808.767409324646, | |
| "size_gb": 0.7898119231685996, | |
| "path": "c4/c4-train.00108-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1875194914, | |
| "size_mb": 1788.3252277374268, | |
| "size_gb": 1.7464113552123308, | |
| "path": "common_crawl/2021-04/en_head_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00957-of-01024.jsonl", | |
| "size": 846413888, | |
| "size_mb": 807.2031860351562, | |
| "size_gb": 0.7882843613624573, | |
| "path": "c4/c4-train.00957-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0119.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1308330413, | |
| "size_mb": 1247.7211122512817, | |
| "size_gb": 1.2184776486828923, | |
| "path": "common_crawl/2020-05/en_middle_0119.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00900-of-01024.jsonl", | |
| "size": 841984860, | |
| "size_mb": 802.9793357849121, | |
| "size_gb": 0.7841595076024532, | |
| "path": "c4/c4-train.00900-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1798576600, | |
| "size_mb": 1715.2563095092773, | |
| "size_gb": 1.6750549897551537, | |
| "path": "common_crawl/2021-04/en_head_0033.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0117.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1285652852, | |
| "size_mb": 1226.0941047668457, | |
| "size_gb": 1.1973575241863728, | |
| "path": "common_crawl/2020-05/en_middle_0117.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00910-of-01024.jsonl", | |
| "size": 844588662, | |
| "size_mb": 805.4625148773193, | |
| "size_gb": 0.7865844871848822, | |
| "path": "c4/c4-train.00910-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00906-of-01024.jsonl", | |
| "size": 844771509, | |
| "size_mb": 805.6368913650513, | |
| "size_gb": 0.7867547767236829, | |
| "path": "c4/c4-train.00906-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00960-of-01024.jsonl", | |
| "size": 843336783, | |
| "size_mb": 804.268630027771, | |
| "size_gb": 0.7854185840114951, | |
| "path": "c4/c4-train.00960-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00911-of-01024.jsonl", | |
| "size": 847352168, | |
| "size_mb": 808.0979995727539, | |
| "size_gb": 0.7891582027077675, | |
| "path": "c4/c4-train.00911-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00964-of-01024.jsonl", | |
| "size": 844939785, | |
| "size_mb": 805.7973718643188, | |
| "size_gb": 0.7869114959612489, | |
| "path": "c4/c4-train.00964-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00969-of-01024.jsonl", | |
| "size": 843853997, | |
| "size_mb": 804.7618837356567, | |
| "size_gb": 0.7859002770856023, | |
| "path": "c4/c4-train.00969-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00958-of-01024.jsonl", | |
| "size": 845610567, | |
| "size_mb": 806.4370794296265, | |
| "size_gb": 0.7875362103804946, | |
| "path": "c4/c4-train.00958-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00971-of-01024.jsonl", | |
| "size": 844379221, | |
| "size_mb": 805.2627763748169, | |
| "size_gb": 0.7863894300535321, | |
| "path": "c4/c4-train.00971-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00962-of-01024.jsonl", | |
| "size": 844732504, | |
| "size_mb": 805.5996932983398, | |
| "size_gb": 0.78671845048666, | |
| "path": "c4/c4-train.00962-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00905-of-01024.jsonl", | |
| "size": 843030748, | |
| "size_mb": 803.9767723083496, | |
| "size_gb": 0.7851335667073727, | |
| "path": "c4/c4-train.00905-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00970-of-01024.jsonl", | |
| "size": 846945057, | |
| "size_mb": 807.7097482681274, | |
| "size_gb": 0.7887790510430932, | |
| "path": "c4/c4-train.00970-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00909-of-01024.jsonl", | |
| "size": 849743603, | |
| "size_mb": 810.3786497116089, | |
| "size_gb": 0.791385400108993, | |
| "path": "c4/c4-train.00909-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00968-of-01024.jsonl", | |
| "size": 845111774, | |
| "size_mb": 805.9613933563232, | |
| "size_gb": 0.7870716731995344, | |
| "path": "c4/c4-train.00968-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00934-of-01024.jsonl", | |
| "size": 848124107, | |
| "size_mb": 808.8341779708862, | |
| "size_gb": 0.7898771269246936, | |
| "path": "c4/c4-train.00934-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_middle_0118.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1269774679, | |
| "size_mb": 1210.9514989852905, | |
| "size_gb": 1.1825698232278228, | |
| "path": "common_crawl/2020-05/en_middle_0118.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00966-of-01024.jsonl", | |
| "size": 846574295, | |
| "size_mb": 807.356162071228, | |
| "size_gb": 0.7884337520226836, | |
| "path": "c4/c4-train.00966-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00963-of-01024.jsonl", | |
| "size": 846654823, | |
| "size_mb": 807.4329595565796, | |
| "size_gb": 0.7885087495669723, | |
| "path": "c4/c4-train.00963-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1861458575, | |
| "size_mb": 1775.2252340316772, | |
| "size_gb": 1.7336183926090598, | |
| "path": "common_crawl/2021-04/en_head_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00985-of-01024.jsonl", | |
| "size": 848910283, | |
| "size_mb": 809.5839338302612, | |
| "size_gb": 0.7906093103811145, | |
| "path": "c4/c4-train.00985-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00986-of-01024.jsonl", | |
| "size": 846640171, | |
| "size_mb": 807.4189863204956, | |
| "size_gb": 0.788495103828609, | |
| "path": "c4/c4-train.00986-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00912-of-01024.jsonl", | |
| "size": 845800380, | |
| "size_mb": 806.6180992126465, | |
| "size_gb": 0.7877129875123501, | |
| "path": "c4/c4-train.00912-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1830346401, | |
| "size_mb": 1745.554352760315, | |
| "size_gb": 1.704642922617495, | |
| "path": "common_crawl/2021-04/en_head_0029.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00967-of-01024.jsonl", | |
| "size": 842082776, | |
| "size_mb": 803.0727157592773, | |
| "size_gb": 0.7842506989836693, | |
| "path": "c4/c4-train.00967-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00929-of-01024.jsonl", | |
| "size": 848052932, | |
| "size_mb": 808.766300201416, | |
| "size_gb": 0.7898108400404453, | |
| "path": "c4/c4-train.00929-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00447-of-01024.jsonl", | |
| "size": 846463712, | |
| "size_mb": 807.2507019042969, | |
| "size_gb": 0.7883307635784149, | |
| "path": "c4/c4-train.00447-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00449-of-01024.jsonl", | |
| "size": 848074503, | |
| "size_mb": 808.7868719100952, | |
| "size_gb": 0.7898309295997024, | |
| "path": "c4/c4-train.00449-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00989-of-01024.jsonl", | |
| "size": 843614691, | |
| "size_mb": 804.5336637496948, | |
| "size_gb": 0.7856774060055614, | |
| "path": "c4/c4-train.00989-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00629-of-01024.jsonl", | |
| "size": 846971366, | |
| "size_mb": 807.7348384857178, | |
| "size_gb": 0.7888035532087088, | |
| "path": "c4/c4-train.00629-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00901-of-01024.jsonl", | |
| "size": 847483251, | |
| "size_mb": 808.2230100631714, | |
| "size_gb": 0.7892802832648158, | |
| "path": "c4/c4-train.00901-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00903-of-01024.jsonl", | |
| "size": 845597407, | |
| "size_mb": 806.4245290756226, | |
| "size_gb": 0.7875239541754127, | |
| "path": "c4/c4-train.00903-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00965-of-01024.jsonl", | |
| "size": 846241706, | |
| "size_mb": 807.0389804840088, | |
| "size_gb": 0.7881240043789148, | |
| "path": "c4/c4-train.00965-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00908-of-01024.jsonl", | |
| "size": 846718229, | |
| "size_mb": 807.4934282302856, | |
| "size_gb": 0.7885678010061383, | |
| "path": "c4/c4-train.00908-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00689-of-01024.jsonl", | |
| "size": 845476892, | |
| "size_mb": 806.3095970153809, | |
| "size_gb": 0.7874117158353329, | |
| "path": "c4/c4-train.00689-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00577-of-01024.jsonl", | |
| "size": 842568017, | |
| "size_mb": 803.5354776382446, | |
| "size_gb": 0.7847026148810983, | |
| "path": "c4/c4-train.00577-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00397-of-01024.jsonl", | |
| "size": 844985143, | |
| "size_mb": 805.8406286239624, | |
| "size_gb": 0.7869537388905883, | |
| "path": "c4/c4-train.00397-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00398-of-01024.jsonl", | |
| "size": 841832536, | |
| "size_mb": 802.8340682983398, | |
| "size_gb": 0.7840176448225975, | |
| "path": "c4/c4-train.00398-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00973-of-01024.jsonl", | |
| "size": 842981692, | |
| "size_mb": 803.929988861084, | |
| "size_gb": 0.7850878797471523, | |
| "path": "c4/c4-train.00973-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00151-of-01024.jsonl", | |
| "size": 835465497, | |
| "size_mb": 796.7619867324829, | |
| "size_gb": 0.7780878776684403, | |
| "path": "c4/c4-train.00151-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00972-of-01024.jsonl", | |
| "size": 847195910, | |
| "size_mb": 807.9489803314209, | |
| "size_gb": 0.7890126761049032, | |
| "path": "c4/c4-train.00972-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00926-of-01024.jsonl", | |
| "size": 849174321, | |
| "size_mb": 809.8357400894165, | |
| "size_gb": 0.7908552149310708, | |
| "path": "c4/c4-train.00926-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00166-of-01024.jsonl", | |
| "size": 847162661, | |
| "size_mb": 807.9172716140747, | |
| "size_gb": 0.7889817105606198, | |
| "path": "c4/c4-train.00166-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00988-of-01024.jsonl", | |
| "size": 848453390, | |
| "size_mb": 809.1482067108154, | |
| "size_gb": 0.7901837956160307, | |
| "path": "c4/c4-train.00988-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00637-of-01024.jsonl", | |
| "size": 844585024, | |
| "size_mb": 805.4590454101562, | |
| "size_gb": 0.7865810990333557, | |
| "path": "c4/c4-train.00637-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00975-of-01024.jsonl", | |
| "size": 846109461, | |
| "size_mb": 806.9128618240356, | |
| "size_gb": 0.7880008416250348, | |
| "path": "c4/c4-train.00975-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00976-of-01024.jsonl", | |
| "size": 845576902, | |
| "size_mb": 806.4049739837646, | |
| "size_gb": 0.7875048574060202, | |
| "path": "c4/c4-train.00976-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00918-of-01024.jsonl", | |
| "size": 843970322, | |
| "size_mb": 804.8728199005127, | |
| "size_gb": 0.7860086131840944, | |
| "path": "c4/c4-train.00918-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00914-of-01024.jsonl", | |
| "size": 848501732, | |
| "size_mb": 809.1943092346191, | |
| "size_gb": 0.7902288176119328, | |
| "path": "c4/c4-train.00914-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00916-of-01024.jsonl", | |
| "size": 846929219, | |
| "size_mb": 807.6946439743042, | |
| "size_gb": 0.7887643007561564, | |
| "path": "c4/c4-train.00916-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00571-of-01024.jsonl", | |
| "size": 843740731, | |
| "size_mb": 804.6538648605347, | |
| "size_gb": 0.7857947899028659, | |
| "path": "c4/c4-train.00571-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00984-of-01024.jsonl", | |
| "size": 846813323, | |
| "size_mb": 807.58411693573, | |
| "size_gb": 0.7886563641950488, | |
| "path": "c4/c4-train.00984-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00978-of-01024.jsonl", | |
| "size": 844734129, | |
| "size_mb": 805.601243019104, | |
| "size_gb": 0.7867199638858438, | |
| "path": "c4/c4-train.00978-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00391-of-01024.jsonl", | |
| "size": 845995736, | |
| "size_mb": 806.8044052124023, | |
| "size_gb": 0.7878949269652367, | |
| "path": "c4/c4-train.00391-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00390-of-01024.jsonl", | |
| "size": 844837467, | |
| "size_mb": 805.6997938156128, | |
| "size_gb": 0.7868162048980594, | |
| "path": "c4/c4-train.00390-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00974-of-01024.jsonl", | |
| "size": 844876361, | |
| "size_mb": 805.7368860244751, | |
| "size_gb": 0.7868524277582765, | |
| "path": "c4/c4-train.00974-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00987-of-01024.jsonl", | |
| "size": 848734201, | |
| "size_mb": 809.4160089492798, | |
| "size_gb": 0.790445321239531, | |
| "path": "c4/c4-train.00987-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00425-of-01024.jsonl", | |
| "size": 845652614, | |
| "size_mb": 806.4771785736084, | |
| "size_gb": 0.7875753697007895, | |
| "path": "c4/c4-train.00425-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00913-of-01024.jsonl", | |
| "size": 845516222, | |
| "size_mb": 806.3471050262451, | |
| "size_gb": 0.7874483447521925, | |
| "path": "c4/c4-train.00913-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00630-of-01024.jsonl", | |
| "size": 843407382, | |
| "size_mb": 804.335958480835, | |
| "size_gb": 0.7854843344539404, | |
| "path": "c4/c4-train.00630-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00915-of-01024.jsonl", | |
| "size": 850199420, | |
| "size_mb": 810.8133506774902, | |
| "size_gb": 0.7918099127709866, | |
| "path": "c4/c4-train.00915-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00612-of-01024.jsonl", | |
| "size": 844963009, | |
| "size_mb": 805.8195199966431, | |
| "size_gb": 0.7869331249967217, | |
| "path": "c4/c4-train.00612-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1831944372, | |
| "size_mb": 1747.078296661377, | |
| "size_gb": 1.706131149083376, | |
| "path": "common_crawl/2021-04/en_head_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00917-of-01024.jsonl", | |
| "size": 846926011, | |
| "size_mb": 807.6915845870972, | |
| "size_gb": 0.7887613130733371, | |
| "path": "c4/c4-train.00917-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00928-of-01024.jsonl", | |
| "size": 845197142, | |
| "size_mb": 806.0428066253662, | |
| "size_gb": 0.7871511783450842, | |
| "path": "c4/c4-train.00928-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00631-of-01024.jsonl", | |
| "size": 845987950, | |
| "size_mb": 806.7969799041748, | |
| "size_gb": 0.7878876756876707, | |
| "path": "c4/c4-train.00631-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00925-of-01024.jsonl", | |
| "size": 844908054, | |
| "size_mb": 805.767110824585, | |
| "size_gb": 0.7868819441646338, | |
| "path": "c4/c4-train.00925-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00919-of-01024.jsonl", | |
| "size": 844946397, | |
| "size_mb": 805.8036775588989, | |
| "size_gb": 0.7869176538661122, | |
| "path": "c4/c4-train.00919-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00668-of-01024.jsonl", | |
| "size": 845906138, | |
| "size_mb": 806.718957901001, | |
| "size_gb": 0.7878114823251963, | |
| "path": "c4/c4-train.00668-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00192-of-01024.jsonl", | |
| "size": 842651579, | |
| "size_mb": 803.6151685714722, | |
| "size_gb": 0.7847804380580783, | |
| "path": "c4/c4-train.00192-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00632-of-01024.jsonl", | |
| "size": 843024512, | |
| "size_mb": 803.9708251953125, | |
| "size_gb": 0.7851277589797974, | |
| "path": "c4/c4-train.00632-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1863650928, | |
| "size_mb": 1777.3160247802734, | |
| "size_gb": 1.7356601804494858, | |
| "path": "common_crawl/2021-04/en_head_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00176-of-01024.jsonl", | |
| "size": 844007049, | |
| "size_mb": 804.9078454971313, | |
| "size_gb": 0.7860428178682923, | |
| "path": "c4/c4-train.00176-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00570-of-01024.jsonl", | |
| "size": 845960263, | |
| "size_mb": 806.7705755233765, | |
| "size_gb": 0.7878618901595473, | |
| "path": "c4/c4-train.00570-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00977-of-01024.jsonl", | |
| "size": 844278857, | |
| "size_mb": 805.1670618057251, | |
| "size_gb": 0.7862959587946534, | |
| "path": "c4/c4-train.00977-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0091.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1384737630, | |
| "size_mb": 1320.5887126922607, | |
| "size_gb": 1.2896374147385359, | |
| "path": "common_crawl/2019-30/en_middle_0091.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00572-of-01024.jsonl", | |
| "size": 843568728, | |
| "size_mb": 804.4898300170898, | |
| "size_gb": 0.7856345996260643, | |
| "path": "c4/c4-train.00572-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00160-of-01024.jsonl", | |
| "size": 845665005, | |
| "size_mb": 806.488995552063, | |
| "size_gb": 0.7875869097188115, | |
| "path": "c4/c4-train.00160-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00609-of-01024.jsonl", | |
| "size": 847067980, | |
| "size_mb": 807.826976776123, | |
| "size_gb": 0.7888935320079327, | |
| "path": "c4/c4-train.00609-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00167-of-01024.jsonl", | |
| "size": 846062180, | |
| "size_mb": 806.8677711486816, | |
| "size_gb": 0.7879568077623844, | |
| "path": "c4/c4-train.00167-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00400-of-01024.jsonl", | |
| "size": 845433641, | |
| "size_mb": 806.268349647522, | |
| "size_gb": 0.7873714352026582, | |
| "path": "c4/c4-train.00400-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00392-of-01024.jsonl", | |
| "size": 844085411, | |
| "size_mb": 804.9825773239136, | |
| "size_gb": 0.7861157981678843, | |
| "path": "c4/c4-train.00392-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00640-of-01024.jsonl", | |
| "size": 846327513, | |
| "size_mb": 807.1208124160767, | |
| "size_gb": 0.7882039183750749, | |
| "path": "c4/c4-train.00640-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00407-of-01024.jsonl", | |
| "size": 849344972, | |
| "size_mb": 809.9984855651855, | |
| "size_gb": 0.7910141460597515, | |
| "path": "c4/c4-train.00407-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00586-of-01024.jsonl", | |
| "size": 845409437, | |
| "size_mb": 806.2452669143677, | |
| "size_gb": 0.7873488934710622, | |
| "path": "c4/c4-train.00586-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00645-of-01024.jsonl", | |
| "size": 849459651, | |
| "size_mb": 810.1078519821167, | |
| "size_gb": 0.7911209492012858, | |
| "path": "c4/c4-train.00645-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00647-of-01024.jsonl", | |
| "size": 846828166, | |
| "size_mb": 807.5982723236084, | |
| "size_gb": 0.7886701878160238, | |
| "path": "c4/c4-train.00647-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00185-of-01024.jsonl", | |
| "size": 849168204, | |
| "size_mb": 809.829906463623, | |
| "size_gb": 0.7908495180308819, | |
| "path": "c4/c4-train.00185-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00651-of-01024.jsonl", | |
| "size": 845564804, | |
| "size_mb": 806.3934364318848, | |
| "size_gb": 0.7874935902655125, | |
| "path": "c4/c4-train.00651-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1894505943, | |
| "size_mb": 1806.741660118103, | |
| "size_gb": 1.764396152459085, | |
| "path": "common_crawl/2021-04/en_head_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00659-of-01024.jsonl", | |
| "size": 845550835, | |
| "size_mb": 806.3801145553589, | |
| "size_gb": 0.7874805806204677, | |
| "path": "c4/c4-train.00659-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00599-of-01024.jsonl", | |
| "size": 846704816, | |
| "size_mb": 807.4806365966797, | |
| "size_gb": 0.788555309176445, | |
| "path": "c4/c4-train.00599-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00171-of-01024.jsonl", | |
| "size": 846321003, | |
| "size_mb": 807.1146039962769, | |
| "size_gb": 0.7881978554651141, | |
| "path": "c4/c4-train.00171-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0090.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1385978557, | |
| "size_mb": 1321.7721529006958, | |
| "size_gb": 1.2907931180670857, | |
| "path": "common_crawl/2019-30/en_middle_0090.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00600-of-01024.jsonl", | |
| "size": 843325798, | |
| "size_mb": 804.2581539154053, | |
| "size_gb": 0.785408353433013, | |
| "path": "c4/c4-train.00600-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00613-of-01024.jsonl", | |
| "size": 848679415, | |
| "size_mb": 809.3637609481812, | |
| "size_gb": 0.7903942978009582, | |
| "path": "c4/c4-train.00613-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00196-of-01024.jsonl", | |
| "size": 846044857, | |
| "size_mb": 806.8512506484985, | |
| "size_gb": 0.7879406744614244, | |
| "path": "c4/c4-train.00196-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00405-of-01024.jsonl", | |
| "size": 841830421, | |
| "size_mb": 802.8320512771606, | |
| "size_gb": 0.7840156750753522, | |
| "path": "c4/c4-train.00405-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00154-of-01024.jsonl", | |
| "size": 848690073, | |
| "size_mb": 809.3739252090454, | |
| "size_gb": 0.7904042238369584, | |
| "path": "c4/c4-train.00154-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00430-of-01024.jsonl", | |
| "size": 845902437, | |
| "size_mb": 806.715428352356, | |
| "size_gb": 0.7878080355003476, | |
| "path": "c4/c4-train.00430-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00184-of-01024.jsonl", | |
| "size": 843687577, | |
| "size_mb": 804.6031732559204, | |
| "size_gb": 0.7857452863827348, | |
| "path": "c4/c4-train.00184-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00927-of-01024.jsonl", | |
| "size": 846418738, | |
| "size_mb": 807.2078113555908, | |
| "size_gb": 0.7882888782769442, | |
| "path": "c4/c4-train.00927-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00622-of-01024.jsonl", | |
| "size": 848638270, | |
| "size_mb": 809.3245220184326, | |
| "size_gb": 0.7903559785336256, | |
| "path": "c4/c4-train.00622-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00607-of-01024.jsonl", | |
| "size": 846255924, | |
| "size_mb": 807.0525398254395, | |
| "size_gb": 0.7881372459232807, | |
| "path": "c4/c4-train.00607-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00433-of-01024.jsonl", | |
| "size": 843059485, | |
| "size_mb": 804.0041780471802, | |
| "size_gb": 0.7851603301241994, | |
| "path": "c4/c4-train.00433-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00633-of-01024.jsonl", | |
| "size": 847284941, | |
| "size_mb": 808.0338869094849, | |
| "size_gb": 0.7890955926850438, | |
| "path": "c4/c4-train.00633-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00426-of-01024.jsonl", | |
| "size": 843861933, | |
| "size_mb": 804.7694520950317, | |
| "size_gb": 0.7859076680615544, | |
| "path": "c4/c4-train.00426-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00420-of-01024.jsonl", | |
| "size": 842873428, | |
| "size_mb": 803.8267402648926, | |
| "size_gb": 0.7849870510399342, | |
| "path": "c4/c4-train.00420-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00611-of-01024.jsonl", | |
| "size": 846683970, | |
| "size_mb": 807.4607563018799, | |
| "size_gb": 0.7885358948260546, | |
| "path": "c4/c4-train.00611-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00431-of-01024.jsonl", | |
| "size": 845206490, | |
| "size_mb": 806.051721572876, | |
| "size_gb": 0.7871598843485117, | |
| "path": "c4/c4-train.00431-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00671-of-01024.jsonl", | |
| "size": 847392973, | |
| "size_mb": 808.1369142532349, | |
| "size_gb": 0.7891962053254247, | |
| "path": "c4/c4-train.00671-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00393-of-01024.jsonl", | |
| "size": 843031382, | |
| "size_mb": 803.9773769378662, | |
| "size_gb": 0.785134157165885, | |
| "path": "c4/c4-train.00393-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00666-of-01024.jsonl", | |
| "size": 846257432, | |
| "size_mb": 807.0539779663086, | |
| "size_gb": 0.7881386503577232, | |
| "path": "c4/c4-train.00666-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00670-of-01024.jsonl", | |
| "size": 848811660, | |
| "size_mb": 809.4898796081543, | |
| "size_gb": 0.7905174605548382, | |
| "path": "c4/c4-train.00670-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00646-of-01024.jsonl", | |
| "size": 846883278, | |
| "size_mb": 807.6508312225342, | |
| "size_gb": 0.788721514865756, | |
| "path": "c4/c4-train.00646-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00573-of-01024.jsonl", | |
| "size": 846306167, | |
| "size_mb": 807.1004552841187, | |
| "size_gb": 0.7881840383633971, | |
| "path": "c4/c4-train.00573-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00682-of-01024.jsonl", | |
| "size": 846105076, | |
| "size_mb": 806.9086799621582, | |
| "size_gb": 0.7879967577755451, | |
| "path": "c4/c4-train.00682-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00673-of-01024.jsonl", | |
| "size": 845550005, | |
| "size_mb": 806.3793230056763, | |
| "size_gb": 0.7874798076227307, | |
| "path": "c4/c4-train.00673-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2061533468, | |
| "size_mb": 1966.0315208435059, | |
| "size_gb": 1.9199526570737362, | |
| "path": "common_crawl/2020-05/en_head_0030.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1984754511, | |
| "size_mb": 1892.809401512146, | |
| "size_gb": 1.848446681164205, | |
| "path": "common_crawl/2020-05/en_head_0007.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2105637501, | |
| "size_mb": 2008.0924043655396, | |
| "size_gb": 1.9610277386382222, | |
| "path": "common_crawl/2020-05/en_head_0034.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0096.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 222740441, | |
| "size_mb": 212.42183780670166, | |
| "size_gb": 0.2074432009831071, | |
| "path": "common_crawl/2019-30/en_middle_0096.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2154309118, | |
| "size_mb": 2054.5092754364014, | |
| "size_gb": 2.0063567142933607, | |
| "path": "common_crawl/2020-05/en_head_0000.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2054079065, | |
| "size_mb": 1958.9224481582642, | |
| "size_gb": 1.9130102032795548, | |
| "path": "common_crawl/2020-05/en_head_0028.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2004950647, | |
| "size_mb": 1912.0699377059937, | |
| "size_gb": 1.8672557985410094, | |
| "path": "common_crawl/2020-05/en_head_0008.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0093.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1407155488, | |
| "size_mb": 1341.9680480957031, | |
| "size_gb": 1.31051567196846, | |
| "path": "common_crawl/2019-30/en_middle_0093.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0092.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1418096595, | |
| "size_mb": 1352.4023008346558, | |
| "size_gb": 1.3207053719088435, | |
| "path": "common_crawl/2019-30/en_middle_0092.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2081366544, | |
| "size_mb": 1984.945816040039, | |
| "size_gb": 1.9384236484766006, | |
| "path": "common_crawl/2020-05/en_head_0035.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2029441050, | |
| "size_mb": 1935.4258060455322, | |
| "size_gb": 1.89006426371634, | |
| "path": "common_crawl/2020-05/en_head_0045.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1863879467, | |
| "size_mb": 1777.5339765548706, | |
| "size_gb": 1.7358730239793658, | |
| "path": "common_crawl/2021-04/en_head_0031.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00177-of-01024.jsonl", | |
| "size": 845939010, | |
| "size_mb": 806.7503070831299, | |
| "size_gb": 0.787842096760869, | |
| "path": "c4/c4-train.00177-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2021018394, | |
| "size_mb": 1927.3933353424072, | |
| "size_gb": 1.8822200540453196, | |
| "path": "common_crawl/2020-05/en_head_0036.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00152-of-01024.jsonl", | |
| "size": 848031609, | |
| "size_mb": 808.7459650039673, | |
| "size_gb": 0.7897909814491868, | |
| "path": "c4/c4-train.00152-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00584-of-01024.jsonl", | |
| "size": 844368249, | |
| "size_mb": 805.2523126602173, | |
| "size_gb": 0.7863792115822434, | |
| "path": "c4/c4-train.00584-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00578-of-01024.jsonl", | |
| "size": 846150383, | |
| "size_mb": 806.9518880844116, | |
| "size_gb": 0.7880389532074332, | |
| "path": "c4/c4-train.00578-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00642-of-01024.jsonl", | |
| "size": 842098064, | |
| "size_mb": 803.0872955322266, | |
| "size_gb": 0.78426493704319, | |
| "path": "c4/c4-train.00642-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00178-of-01024.jsonl", | |
| "size": 844378739, | |
| "size_mb": 805.2623167037964, | |
| "size_gb": 0.7863889811560512, | |
| "path": "c4/c4-train.00178-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00408-of-01024.jsonl", | |
| "size": 844426409, | |
| "size_mb": 805.3077783584595, | |
| "size_gb": 0.7864333773031831, | |
| "path": "c4/c4-train.00408-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00401-of-01024.jsonl", | |
| "size": 842889604, | |
| "size_mb": 803.8421669006348, | |
| "size_gb": 0.7850021161139011, | |
| "path": "c4/c4-train.00401-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00574-of-01024.jsonl", | |
| "size": 844427445, | |
| "size_mb": 805.3087663650513, | |
| "size_gb": 0.7864343421533704, | |
| "path": "c4/c4-train.00574-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00588-of-01024.jsonl", | |
| "size": 846519613, | |
| "size_mb": 807.3040132522583, | |
| "size_gb": 0.7883828254416585, | |
| "path": "c4/c4-train.00588-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00399-of-01024.jsonl", | |
| "size": 847082589, | |
| "size_mb": 807.8409090042114, | |
| "size_gb": 0.7889071376994252, | |
| "path": "c4/c4-train.00399-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00411-of-01024.jsonl", | |
| "size": 843792462, | |
| "size_mb": 804.7031993865967, | |
| "size_gb": 0.7858429681509733, | |
| "path": "c4/c4-train.00411-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00644-of-01024.jsonl", | |
| "size": 844967303, | |
| "size_mb": 805.8236150741577, | |
| "size_gb": 0.7869371240958571, | |
| "path": "c4/c4-train.00644-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00199-of-01024.jsonl", | |
| "size": 850706862, | |
| "size_mb": 811.297285079956, | |
| "size_gb": 0.7922825049608946, | |
| "path": "c4/c4-train.00199-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00581-of-01024.jsonl", | |
| "size": 848790526, | |
| "size_mb": 809.4697246551514, | |
| "size_gb": 0.7904977779835463, | |
| "path": "c4/c4-train.00581-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00580-of-01024.jsonl", | |
| "size": 847927624, | |
| "size_mb": 808.6467971801758, | |
| "size_gb": 0.7896941378712654, | |
| "path": "c4/c4-train.00580-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00638-of-01024.jsonl", | |
| "size": 844500284, | |
| "size_mb": 805.378231048584, | |
| "size_gb": 0.7865021787583828, | |
| "path": "c4/c4-train.00638-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00592-of-01024.jsonl", | |
| "size": 849421279, | |
| "size_mb": 810.0712575912476, | |
| "size_gb": 0.7910852124914527, | |
| "path": "c4/c4-train.00592-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00639-of-01024.jsonl", | |
| "size": 848736025, | |
| "size_mb": 809.4177484512329, | |
| "size_gb": 0.7904470199719071, | |
| "path": "c4/c4-train.00639-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00648-of-01024.jsonl", | |
| "size": 840467090, | |
| "size_mb": 801.5318775177002, | |
| "size_gb": 0.7827459741383791, | |
| "path": "c4/c4-train.00648-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00634-of-01024.jsonl", | |
| "size": 842199081, | |
| "size_mb": 803.183632850647, | |
| "size_gb": 0.7843590164557099, | |
| "path": "c4/c4-train.00634-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00412-of-01024.jsonl", | |
| "size": 840056990, | |
| "size_mb": 801.140775680542, | |
| "size_gb": 0.7823640387505293, | |
| "path": "c4/c4-train.00412-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00591-of-01024.jsonl", | |
| "size": 846761476, | |
| "size_mb": 807.5346717834473, | |
| "size_gb": 0.7886080779135227, | |
| "path": "c4/c4-train.00591-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00165-of-01024.jsonl", | |
| "size": 848936423, | |
| "size_mb": 809.6088628768921, | |
| "size_gb": 0.7906336551532149, | |
| "path": "c4/c4-train.00165-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00636-of-01024.jsonl", | |
| "size": 842862859, | |
| "size_mb": 803.8166608810425, | |
| "size_gb": 0.784977207891643, | |
| "path": "c4/c4-train.00636-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00394-of-01024.jsonl", | |
| "size": 846414616, | |
| "size_mb": 807.2038803100586, | |
| "size_gb": 0.7882850393652916, | |
| "path": "c4/c4-train.00394-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2064866351, | |
| "size_mb": 1969.2100057601929, | |
| "size_gb": 1.9230566462501884, | |
| "path": "common_crawl/2020-05/en_head_0004.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2116023488, | |
| "size_mb": 2017.9972534179688, | |
| "size_gb": 1.970700442790985, | |
| "path": "common_crawl/2020-05/en_head_0009.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00641-of-01024.jsonl", | |
| "size": 845716559, | |
| "size_mb": 806.538161277771, | |
| "size_gb": 0.7876349231228232, | |
| "path": "c4/c4-train.00641-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00582-of-01024.jsonl", | |
| "size": 846198163, | |
| "size_mb": 806.9974546432495, | |
| "size_gb": 0.7880834518000484, | |
| "path": "c4/c4-train.00582-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00187-of-01024.jsonl", | |
| "size": 847014957, | |
| "size_mb": 807.7764101028442, | |
| "size_gb": 0.7888441504910588, | |
| "path": "c4/c4-train.00187-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2067795465, | |
| "size_mb": 1972.0034265518188, | |
| "size_gb": 1.9257845962420106, | |
| "path": "common_crawl/2020-05/en_head_0001.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00603-of-01024.jsonl", | |
| "size": 846788833, | |
| "size_mb": 807.5607614517212, | |
| "size_gb": 0.7886335561051965, | |
| "path": "c4/c4-train.00603-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00678-of-01024.jsonl", | |
| "size": 845521423, | |
| "size_mb": 806.3520650863647, | |
| "size_gb": 0.7874531885609031, | |
| "path": "c4/c4-train.00678-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00402-of-01024.jsonl", | |
| "size": 846278434, | |
| "size_mb": 807.0740070343018, | |
| "size_gb": 0.7881582099944353, | |
| "path": "c4/c4-train.00402-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00170-of-01024.jsonl", | |
| "size": 841607829, | |
| "size_mb": 802.6197710037231, | |
| "size_gb": 0.7838083701208234, | |
| "path": "c4/c4-train.00170-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00663-of-01024.jsonl", | |
| "size": 842949079, | |
| "size_mb": 803.898886680603, | |
| "size_gb": 0.7850575065240264, | |
| "path": "c4/c4-train.00663-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00193-of-01024.jsonl", | |
| "size": 848772332, | |
| "size_mb": 809.4523735046387, | |
| "size_gb": 0.7904808335006237, | |
| "path": "c4/c4-train.00193-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0094.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1390833763, | |
| "size_mb": 1326.4024381637573, | |
| "size_gb": 1.2953148810192943, | |
| "path": "common_crawl/2019-30/en_middle_0094.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2058957268, | |
| "size_mb": 1963.57466506958, | |
| "size_gb": 1.9175533838570118, | |
| "path": "common_crawl/2020-05/en_head_0011.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00585-of-01024.jsonl", | |
| "size": 848695400, | |
| "size_mb": 809.3790054321289, | |
| "size_gb": 0.7904091849923134, | |
| "path": "c4/c4-train.00585-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00186-of-01024.jsonl", | |
| "size": 844110777, | |
| "size_mb": 805.0067682266235, | |
| "size_gb": 0.786139422096312, | |
| "path": "c4/c4-train.00186-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00587-of-01024.jsonl", | |
| "size": 849570570, | |
| "size_mb": 810.2136325836182, | |
| "size_gb": 0.7912242505699396, | |
| "path": "c4/c4-train.00587-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00409-of-01024.jsonl", | |
| "size": 845946422, | |
| "size_mb": 806.7573757171631, | |
| "size_gb": 0.7878489997237921, | |
| "path": "c4/c4-train.00409-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00156-of-01024.jsonl", | |
| "size": 847789064, | |
| "size_mb": 808.5146560668945, | |
| "size_gb": 0.7895650938153267, | |
| "path": "c4/c4-train.00156-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00189-of-01024.jsonl", | |
| "size": 839838563, | |
| "size_mb": 800.9324674606323, | |
| "size_gb": 0.7821606127545238, | |
| "path": "c4/c4-train.00189-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00664-of-01024.jsonl", | |
| "size": 845366832, | |
| "size_mb": 806.2046356201172, | |
| "size_gb": 0.7873092144727707, | |
| "path": "c4/c4-train.00664-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00605-of-01024.jsonl", | |
| "size": 846773521, | |
| "size_mb": 807.5461587905884, | |
| "size_gb": 0.788619295693934, | |
| "path": "c4/c4-train.00605-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00423-of-01024.jsonl", | |
| "size": 846338053, | |
| "size_mb": 807.1308641433716, | |
| "size_gb": 0.7882137345150113, | |
| "path": "c4/c4-train.00423-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00672-of-01024.jsonl", | |
| "size": 843280412, | |
| "size_mb": 804.2148704528809, | |
| "size_gb": 0.7853660844266415, | |
| "path": "c4/c4-train.00672-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00413-of-01024.jsonl", | |
| "size": 844393501, | |
| "size_mb": 805.2763948440552, | |
| "size_gb": 0.7864027293398976, | |
| "path": "c4/c4-train.00413-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00650-of-01024.jsonl", | |
| "size": 846976351, | |
| "size_mb": 807.7395925521851, | |
| "size_gb": 0.7888081958517432, | |
| "path": "c4/c4-train.00650-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00421-of-01024.jsonl", | |
| "size": 846148291, | |
| "size_mb": 806.9498929977417, | |
| "size_gb": 0.7880370048806071, | |
| "path": "c4/c4-train.00421-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00652-of-01024.jsonl", | |
| "size": 839799833, | |
| "size_mb": 800.8955316543579, | |
| "size_gb": 0.7821245426312089, | |
| "path": "c4/c4-train.00652-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00209-of-01024.jsonl", | |
| "size": 848061363, | |
| "size_mb": 808.7743406295776, | |
| "size_gb": 0.7898186920210719, | |
| "path": "c4/c4-train.00209-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00419-of-01024.jsonl", | |
| "size": 847882348, | |
| "size_mb": 808.6036186218262, | |
| "size_gb": 0.7896519713103771, | |
| "path": "c4/c4-train.00419-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00649-of-01024.jsonl", | |
| "size": 845587872, | |
| "size_mb": 806.4154357910156, | |
| "size_gb": 0.7875150740146637, | |
| "path": "c4/c4-train.00649-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00594-of-01024.jsonl", | |
| "size": 843680754, | |
| "size_mb": 804.5966663360596, | |
| "size_gb": 0.7857389319688082, | |
| "path": "c4/c4-train.00594-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00410-of-01024.jsonl", | |
| "size": 844981770, | |
| "size_mb": 805.8374118804932, | |
| "size_gb": 0.7869505975395441, | |
| "path": "c4/c4-train.00410-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00153-of-01024.jsonl", | |
| "size": 843262333, | |
| "size_mb": 804.1976289749146, | |
| "size_gb": 0.785349247045815, | |
| "path": "c4/c4-train.00153-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00432-of-01024.jsonl", | |
| "size": 844333312, | |
| "size_mb": 805.218994140625, | |
| "size_gb": 0.7863466739654541, | |
| "path": "c4/c4-train.00432-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00665-of-01024.jsonl", | |
| "size": 850408268, | |
| "size_mb": 811.012523651123, | |
| "size_gb": 0.7920044176280499, | |
| "path": "c4/c4-train.00665-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00604-of-01024.jsonl", | |
| "size": 845681392, | |
| "size_mb": 806.5046234130859, | |
| "size_gb": 0.7876021713018417, | |
| "path": "c4/c4-train.00604-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00662-of-01024.jsonl", | |
| "size": 842624800, | |
| "size_mb": 803.5896301269531, | |
| "size_gb": 0.7847554981708527, | |
| "path": "c4/c4-train.00662-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00660-of-01024.jsonl", | |
| "size": 847935638, | |
| "size_mb": 808.6544399261475, | |
| "size_gb": 0.7897016014903784, | |
| "path": "c4/c4-train.00660-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00172-of-01024.jsonl", | |
| "size": 845339341, | |
| "size_mb": 806.1784181594849, | |
| "size_gb": 0.7872836114838719, | |
| "path": "c4/c4-train.00172-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00169-of-01024.jsonl", | |
| "size": 847442273, | |
| "size_mb": 808.1839303970337, | |
| "size_gb": 0.7892421195283532, | |
| "path": "c4/c4-train.00169-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00173-of-01024.jsonl", | |
| "size": 847014587, | |
| "size_mb": 807.7760572433472, | |
| "size_gb": 0.7888438059017062, | |
| "path": "c4/c4-train.00173-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00168-of-01024.jsonl", | |
| "size": 846807642, | |
| "size_mb": 807.5786991119385, | |
| "size_gb": 0.7886510733515024, | |
| "path": "c4/c4-train.00168-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00201-of-01024.jsonl", | |
| "size": 842134215, | |
| "size_mb": 803.121771812439, | |
| "size_gb": 0.7842986052855849, | |
| "path": "c4/c4-train.00201-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00661-of-01024.jsonl", | |
| "size": 842182348, | |
| "size_mb": 803.1676750183105, | |
| "size_gb": 0.7843434326350689, | |
| "path": "c4/c4-train.00661-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00590-of-01024.jsonl", | |
| "size": 845161887, | |
| "size_mb": 806.0091848373413, | |
| "size_gb": 0.7871183445677161, | |
| "path": "c4/c4-train.00590-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00674-of-01024.jsonl", | |
| "size": 844375375, | |
| "size_mb": 805.259108543396, | |
| "size_gb": 0.7863858481869102, | |
| "path": "c4/c4-train.00674-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00422-of-01024.jsonl", | |
| "size": 848292205, | |
| "size_mb": 808.9944887161255, | |
| "size_gb": 0.7900336803868413, | |
| "path": "c4/c4-train.00422-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00589-of-01024.jsonl", | |
| "size": 847899994, | |
| "size_mb": 808.6204471588135, | |
| "size_gb": 0.7896684054285288, | |
| "path": "c4/c4-train.00589-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00424-of-01024.jsonl", | |
| "size": 841718597, | |
| "size_mb": 802.7254076004028, | |
| "size_gb": 0.7839115308597684, | |
| "path": "c4/c4-train.00424-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00434-of-01024.jsonl", | |
| "size": 848799281, | |
| "size_mb": 809.4780740737915, | |
| "size_gb": 0.790505931712687, | |
| "path": "c4/c4-train.00434-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00602-of-01024.jsonl", | |
| "size": 846430425, | |
| "size_mb": 807.2189569473267, | |
| "size_gb": 0.7882997626438737, | |
| "path": "c4/c4-train.00602-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00406-of-01024.jsonl", | |
| "size": 842212263, | |
| "size_mb": 803.1962041854858, | |
| "size_gb": 0.7843712931498885, | |
| "path": "c4/c4-train.00406-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00175-of-01024.jsonl", | |
| "size": 843821134, | |
| "size_mb": 804.7305431365967, | |
| "size_gb": 0.7858696710318327, | |
| "path": "c4/c4-train.00175-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00428-of-01024.jsonl", | |
| "size": 844781877, | |
| "size_mb": 805.6467790603638, | |
| "size_gb": 0.7867644326761365, | |
| "path": "c4/c4-train.00428-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00174-of-01024.jsonl", | |
| "size": 845454027, | |
| "size_mb": 806.2877912521362, | |
| "size_gb": 0.7873904211446643, | |
| "path": "c4/c4-train.00174-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1982730798, | |
| "size_mb": 1890.8794384002686, | |
| "size_gb": 1.8465619515627623, | |
| "path": "common_crawl/2020-05/en_head_0026.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00163-of-01024.jsonl", | |
| "size": 849334132, | |
| "size_mb": 809.9881477355957, | |
| "size_gb": 0.7910040505230427, | |
| "path": "c4/c4-train.00163-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00202-of-01024.jsonl", | |
| "size": 847297053, | |
| "size_mb": 808.0454378128052, | |
| "size_gb": 0.7891068728640676, | |
| "path": "c4/c4-train.00202-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00614-of-01024.jsonl", | |
| "size": 849094628, | |
| "size_mb": 809.7597389221191, | |
| "size_gb": 0.790780995041132, | |
| "path": "c4/c4-train.00614-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00601-of-01024.jsonl", | |
| "size": 846012180, | |
| "size_mb": 806.8200874328613, | |
| "size_gb": 0.7879102416336536, | |
| "path": "c4/c4-train.00601-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2019325505, | |
| "size_mb": 1925.7788705825806, | |
| "size_gb": 1.8806434283033013, | |
| "path": "common_crawl/2020-05/en_head_0021.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2082184179, | |
| "size_mb": 1985.725573539734, | |
| "size_gb": 1.9391851304098964, | |
| "path": "common_crawl/2020-05/en_head_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00615-of-01024.jsonl", | |
| "size": 839858170, | |
| "size_mb": 800.9511661529541, | |
| "size_gb": 0.7821788731962442, | |
| "path": "c4/c4-train.00615-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00197-of-01024.jsonl", | |
| "size": 849303393, | |
| "size_mb": 809.9588327407837, | |
| "size_gb": 0.7909754225984216, | |
| "path": "c4/c4-train.00197-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00677-of-01024.jsonl", | |
| "size": 845916645, | |
| "size_mb": 806.7289781570435, | |
| "size_gb": 0.7878212677314878, | |
| "path": "c4/c4-train.00677-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00676-of-01024.jsonl", | |
| "size": 845556054, | |
| "size_mb": 806.3850917816162, | |
| "size_gb": 0.7874854411929846, | |
| "path": "c4/c4-train.00676-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00616-of-01024.jsonl", | |
| "size": 848655557, | |
| "size_mb": 809.3410081863403, | |
| "size_gb": 0.790372078306973, | |
| "path": "c4/c4-train.00616-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2097372940, | |
| "size_mb": 2000.2107048034668, | |
| "size_gb": 1.9533307664096355, | |
| "path": "common_crawl/2020-05/en_head_0025.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2028491655, | |
| "size_mb": 1934.5203924179077, | |
| "size_gb": 1.889180070720613, | |
| "path": "common_crawl/2020-05/en_head_0002.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2085513593, | |
| "size_mb": 1988.9007501602173, | |
| "size_gb": 1.9422858888283372, | |
| "path": "common_crawl/2020-05/en_head_0010.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00188-of-01024.jsonl", | |
| "size": 846799268, | |
| "size_mb": 807.5707130432129, | |
| "size_gb": 0.7886432744562626, | |
| "path": "c4/c4-train.00188-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2080671577, | |
| "size_mb": 1984.2830438613892, | |
| "size_gb": 1.9377764100208879, | |
| "path": "common_crawl/2020-05/en_head_0048.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2054413397, | |
| "size_mb": 1959.241291999817, | |
| "size_gb": 1.9133215742185712, | |
| "path": "common_crawl/2020-05/en_head_0012.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00205-of-01024.jsonl", | |
| "size": 848033857, | |
| "size_mb": 808.7481088638306, | |
| "size_gb": 0.7897930750623345, | |
| "path": "c4/c4-train.00205-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00203-of-01024.jsonl", | |
| "size": 848269091, | |
| "size_mb": 808.9724454879761, | |
| "size_gb": 0.7900121537968516, | |
| "path": "c4/c4-train.00203-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00198-of-01024.jsonl", | |
| "size": 849384073, | |
| "size_mb": 810.0357751846313, | |
| "size_gb": 0.7910505617037416, | |
| "path": "c4/c4-train.00198-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00679-of-01024.jsonl", | |
| "size": 848779203, | |
| "size_mb": 809.4589262008667, | |
| "size_gb": 0.7904872326180339, | |
| "path": "c4/c4-train.00679-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00179-of-01024.jsonl", | |
| "size": 846643018, | |
| "size_mb": 807.4217014312744, | |
| "size_gb": 0.7884977553039789, | |
| "path": "c4/c4-train.00179-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00440-of-01024.jsonl", | |
| "size": 846670955, | |
| "size_mb": 807.4483442306519, | |
| "size_gb": 0.788523773662746, | |
| "path": "c4/c4-train.00440-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00620-of-01024.jsonl", | |
| "size": 849724002, | |
| "size_mb": 810.359956741333, | |
| "size_gb": 0.791367145255208, | |
| "path": "c4/c4-train.00620-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00437-of-01024.jsonl", | |
| "size": 847489834, | |
| "size_mb": 808.2292881011963, | |
| "size_gb": 0.7892864141613245, | |
| "path": "c4/c4-train.00437-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00441-of-01024.jsonl", | |
| "size": 842399203, | |
| "size_mb": 803.3744840621948, | |
| "size_gb": 0.7845453945919871, | |
| "path": "c4/c4-train.00441-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00619-of-01024.jsonl", | |
| "size": 845354412, | |
| "size_mb": 806.1927909851074, | |
| "size_gb": 0.787297647446394, | |
| "path": "c4/c4-train.00619-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00438-of-01024.jsonl", | |
| "size": 849342000, | |
| "size_mb": 809.9956512451172, | |
| "size_gb": 0.7910113781690598, | |
| "path": "c4/c4-train.00438-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2011774419, | |
| "size_mb": 1918.5775938034058, | |
| "size_gb": 1.8736109314486384, | |
| "path": "common_crawl/2020-05/en_head_0027.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2062708842, | |
| "size_mb": 1967.1524448394775, | |
| "size_gb": 1.9210473094135523, | |
| "path": "common_crawl/2020-05/en_head_0003.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00164-of-01024.jsonl", | |
| "size": 848299683, | |
| "size_mb": 809.0016202926636, | |
| "size_gb": 0.7900406448170543, | |
| "path": "c4/c4-train.00164-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00621-of-01024.jsonl", | |
| "size": 844326158, | |
| "size_mb": 805.2121715545654, | |
| "size_gb": 0.7863400112837553, | |
| "path": "c4/c4-train.00621-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00681-of-01024.jsonl", | |
| "size": 848761701, | |
| "size_mb": 809.442234992981, | |
| "size_gb": 0.790470932610333, | |
| "path": "c4/c4-train.00681-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00617-of-01024.jsonl", | |
| "size": 847355169, | |
| "size_mb": 808.1008615493774, | |
| "size_gb": 0.7891609976068139, | |
| "path": "c4/c4-train.00617-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2056861083, | |
| "size_mb": 1961.575587272644, | |
| "size_gb": 1.9156011594459414, | |
| "path": "common_crawl/2020-05/en_head_0024.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00623-of-01024.jsonl", | |
| "size": 849301287, | |
| "size_mb": 809.9568243026733, | |
| "size_gb": 0.7909734612330794, | |
| "path": "c4/c4-train.00623-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00442-of-01024.jsonl", | |
| "size": 847542414, | |
| "size_mb": 808.2794322967529, | |
| "size_gb": 0.7893353831022978, | |
| "path": "c4/c4-train.00442-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00618-of-01024.jsonl", | |
| "size": 844145823, | |
| "size_mb": 805.0401906967163, | |
| "size_gb": 0.786172061227262, | |
| "path": "c4/c4-train.00618-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2000089322, | |
| "size_mb": 1907.43381690979, | |
| "size_gb": 1.8627283368259668, | |
| "path": "common_crawl/2020-05/en_head_0023.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00439-of-01024.jsonl", | |
| "size": 843071697, | |
| "size_mb": 804.0158243179321, | |
| "size_gb": 0.7851717034354806, | |
| "path": "c4/c4-train.00439-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00436-of-01024.jsonl", | |
| "size": 845638265, | |
| "size_mb": 806.4634943008423, | |
| "size_gb": 0.7875620061531663, | |
| "path": "c4/c4-train.00436-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00683-of-01024.jsonl", | |
| "size": 848492798, | |
| "size_mb": 809.1857891082764, | |
| "size_gb": 0.7902204971760511, | |
| "path": "c4/c4-train.00683-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2066643894, | |
| "size_mb": 1970.9052028656006, | |
| "size_gb": 1.924712112173438, | |
| "path": "common_crawl/2020-05/en_head_0014.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2056515447, | |
| "size_mb": 1961.2459630966187, | |
| "size_gb": 1.9152792608365417, | |
| "path": "common_crawl/2020-05/en_head_0046.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2140791654, | |
| "size_mb": 2041.6180171966553, | |
| "size_gb": 1.9937675949186087, | |
| "path": "common_crawl/2020-05/en_head_0044.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2046357626, | |
| "size_mb": 1951.5587100982666, | |
| "size_gb": 1.9058190528303385, | |
| "path": "common_crawl/2020-05/en_head_0040.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1999665531, | |
| "size_mb": 1907.029658317566, | |
| "size_gb": 1.862333650700748, | |
| "path": "common_crawl/2020-05/en_head_0051.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2062428997, | |
| "size_mb": 1966.8855638504028, | |
| "size_gb": 1.920786683447659, | |
| "path": "common_crawl/2020-05/en_head_0032.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00680-of-01024.jsonl", | |
| "size": 844004233, | |
| "size_mb": 804.9051599502563, | |
| "size_gb": 0.7860401952639222, | |
| "path": "c4/c4-train.00680-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2006706896, | |
| "size_mb": 1913.7448272705078, | |
| "size_gb": 1.8688914328813553, | |
| "path": "common_crawl/2020-05/en_head_0042.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2002952798, | |
| "size_mb": 1910.1646404266357, | |
| "size_gb": 1.8653951566666365, | |
| "path": "common_crawl/2020-05/en_head_0022.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2061209501, | |
| "size_mb": 1965.7225618362427, | |
| "size_gb": 1.9196509392932057, | |
| "path": "common_crawl/2020-05/en_head_0043.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00180-of-01024.jsonl", | |
| "size": 847450715, | |
| "size_mb": 808.1919813156128, | |
| "size_gb": 0.7892499817535281, | |
| "path": "c4/c4-train.00180-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00445-of-01024.jsonl", | |
| "size": 849716248, | |
| "size_mb": 810.3525619506836, | |
| "size_gb": 0.7913599237799644, | |
| "path": "c4/c4-train.00445-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00627-of-01024.jsonl", | |
| "size": 848769067, | |
| "size_mb": 809.4492597579956, | |
| "size_gb": 0.7904777927324176, | |
| "path": "c4/c4-train.00627-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00685-of-01024.jsonl", | |
| "size": 844704239, | |
| "size_mb": 805.5727376937866, | |
| "size_gb": 0.7866921266540885, | |
| "path": "c4/c4-train.00685-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00624-of-01024.jsonl", | |
| "size": 848537095, | |
| "size_mb": 809.2280340194702, | |
| "size_gb": 0.7902617519721389, | |
| "path": "c4/c4-train.00624-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1987904146, | |
| "size_mb": 1895.8131275177002, | |
| "size_gb": 1.851380007341504, | |
| "path": "common_crawl/2020-05/en_head_0041.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00443-of-01024.jsonl", | |
| "size": 843706183, | |
| "size_mb": 804.6209173202515, | |
| "size_gb": 0.7857626145705581, | |
| "path": "c4/c4-train.00443-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2021-04/en_head_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1857165100, | |
| "size_mb": 1771.130657196045, | |
| "size_gb": 1.7296197824180126, | |
| "path": "common_crawl/2021-04/en_head_0013.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2021-04" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1981167326, | |
| "size_mb": 1889.3883953094482, | |
| "size_gb": 1.845105854794383, | |
| "path": "common_crawl/2020-05/en_head_0052.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2064700150, | |
| "size_mb": 1969.0515041351318, | |
| "size_gb": 1.9229018595069647, | |
| "path": "common_crawl/2020-05/en_head_0015.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2042621320, | |
| "size_mb": 1947.995491027832, | |
| "size_gb": 1.9023393467068672, | |
| "path": "common_crawl/2020-05/en_head_0039.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2081867148, | |
| "size_mb": 1985.4232292175293, | |
| "size_gb": 1.9388898722827435, | |
| "path": "common_crawl/2020-05/en_head_0047.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00686-of-01024.jsonl", | |
| "size": 847111998, | |
| "size_mb": 807.8689556121826, | |
| "size_gb": 0.7889345269650221, | |
| "path": "c4/c4-train.00686-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00162-of-01024.jsonl", | |
| "size": 848685593, | |
| "size_mb": 809.3696527481079, | |
| "size_gb": 0.7904000515118241, | |
| "path": "c4/c4-train.00162-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00684-of-01024.jsonl", | |
| "size": 845175993, | |
| "size_mb": 806.0226373672485, | |
| "size_gb": 0.7871314818039536, | |
| "path": "c4/c4-train.00684-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00444-of-01024.jsonl", | |
| "size": 844914713, | |
| "size_mb": 805.7734613418579, | |
| "size_gb": 0.7868881458416581, | |
| "path": "c4/c4-train.00444-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00155-of-01024.jsonl", | |
| "size": 842470546, | |
| "size_mb": 803.4425220489502, | |
| "size_gb": 0.7846118379384279, | |
| "path": "c4/c4-train.00155-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2020-05/en_head_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 2069381768, | |
| "size_mb": 1973.516242980957, | |
| "size_gb": 1.9272619560360909, | |
| "path": "common_crawl/2020-05/en_head_0050.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2020-05" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00626-of-01024.jsonl", | |
| "size": 849400046, | |
| "size_mb": 810.0510082244873, | |
| "size_gb": 0.7910654377192259, | |
| "path": "c4/c4-train.00626-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00628-of-01024.jsonl", | |
| "size": 849728262, | |
| "size_mb": 810.3640193939209, | |
| "size_gb": 0.7913711126893759, | |
| "path": "c4/c4-train.00628-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00625-of-01024.jsonl", | |
| "size": 840851580, | |
| "size_mb": 801.8985557556152, | |
| "size_gb": 0.783104058355093, | |
| "path": "c4/c4-train.00625-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00446-of-01024.jsonl", | |
| "size": 847429584, | |
| "size_mb": 808.1718292236328, | |
| "size_gb": 0.7892303019762039, | |
| "path": "c4/c4-train.00446-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00688-of-01024.jsonl", | |
| "size": 846220912, | |
| "size_mb": 807.0191497802734, | |
| "size_gb": 0.7881046384572983, | |
| "path": "c4/c4-train.00688-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00687-of-01024.jsonl", | |
| "size": 845421625, | |
| "size_mb": 806.256890296936, | |
| "size_gb": 0.7873602444306016, | |
| "path": "c4/c4-train.00687-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00194-of-01024.jsonl", | |
| "size": 851656892, | |
| "size_mb": 812.2033042907715, | |
| "size_gb": 0.7931672893464565, | |
| "path": "c4/c4-train.00194-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00181-of-01024.jsonl", | |
| "size": 850148860, | |
| "size_mb": 810.7651329040527, | |
| "size_gb": 0.791762825101614, | |
| "path": "c4/c4-train.00181-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/c4/c4-train.00448-of-01024.jsonl", | |
| "size": 849254597, | |
| "size_mb": 809.9122972488403, | |
| "size_gb": 0.7909299777820706, | |
| "path": "c4/c4-train.00448-of-01024.jsonl", | |
| "top_folder": "c4", | |
| "top_folders": "c4" | |
| }, | |
| { | |
| "url": "https://data.together.xyz/redpajama-data-1T/v1.0.0/common_crawl/2019-30/en_middle_0075.json.gz.dedup.classifier.jsonl.zst", | |
| "size": 1430134008, | |
| "size_mb": 1363.8820724487305, | |
| "size_gb": 1.3319160863757133, | |
| "path": "common_crawl/2019-30/en_middle_0075.json.gz.dedup.classifier.jsonl.zst", | |
| "top_folder": "common_crawl", | |
| "top_folders": "common_crawl/2019-30" | |
| } | |
| ] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment