Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Scripts related to 2020-04 ooni.org blog post
#!/bin/sh
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-03-03/20200303T173454Z-IR-AS58224-sni_blocking-20200303T173455Z_AS58224_NsYh7KbWaHwLRszVEjdjPB0KMrKyNKqLmLkvuNlF8rgYclnawb-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-03-03/20200303T173918Z-IR-AS58224-sni_blocking-20200303T173918Z_AS58224_6uCcSkGisBYuqARw6HEUuQKphw2eNvGGG4N0QjjJ8dCL4gA56o-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-03-03/20200303T174133Z-IR-AS58224-sni_blocking-20200303T174133Z_AS58224_0E6vdfilpdB8Rgo93zAaMEJu0Qv6o3jO08Ruqg6tgCvlZpYsk7-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-03-03/20200303T174357Z-IR-AS58224-sni_blocking-20200303T174358Z_AS58224_NSHgV5FSLuddMoJBQ1rBfw0S9WFw0uPFluJxg2wbde8aV3DoJ6-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-03-03/20200303T175032Z-IR-AS58224-sni_blocking-20200303T175032Z_AS58224_JUFxADDxYSHOCN42eSWdQituqymwCYkV8pH2qJl7ktfplV6y02-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-03-04/20200304T115409Z-IR-AS58224-sni_blocking-20200304T115410Z_AS58224_4uMSW6x57zQ7nLTYrDZZzedMCp5FtqqM4KtKOTkudVA5aiZVnM-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-09/20200409T202516Z-IR-AS58224-sni_blocking-20200409T202518Z_AS58224_OmEMQ2bMWGPT7hB4FQ5j75xZ82jyjb0pOqbjr0lgBJ0vlMsmu0-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-09/20200409T211913Z-IR-AS58224-sni_blocking-20200409T211914Z_AS58224_d1P4Q1RuNqtkKhLtADu7igw4DVAO69re0Ha4lYdd9Ljo9zdd2y-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-09/20200409T212124Z-IR-AS58224-sni_blocking-20200409T212126Z_AS58224_VLbU8aJp5mMN0pZb1VFyJTksWVQgKNhFNGAakXA9ORjVujmNN7-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-09/20200409T212357Z-IR-AS58224-sni_blocking-20200409T212359Z_AS58224_68gpZT7VIW6ZUjUb5yEbUH54JxXqRc4C32SDinHLYgemf0A0Dv-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-09/20200409T212444Z-IR-AS58224-sni_blocking-20200409T212445Z_AS58224_fOoYyTjJTdzOPcOaLyDNmZocVNdhMprlcOoxTJs5aTPAtWmnVr-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-09/20200409T212703Z-IR-AS197207-sni_blocking-20200409T212704Z_AS197207_qW4akPWGomyHzXhtOCnYHImapSGUbpkUtjnbus1QBfdA85yNMN-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-09/20200409T212802Z-IR-AS197207-sni_blocking-20200409T212803Z_AS197207_0n7IWTsLXS9YXh0yT1CzKew7ALV8Oh2UnB6r4feBhKA5e6paNO-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-09/20200409T212848Z-IR-AS197207-sni_blocking-20200409T212849Z_AS197207_OfhR1mERPO03ViUIo47thtzjoCPbwNROqD9BMYFVJRPuvTZM5G-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-09/20200409T212931Z-IR-AS197207-sni_blocking-20200409T212932Z_AS197207_mNqcAZghZJ93QFfdF2deMfgxhjaQry6CWjZfjhIL6IDTFglyKI-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-09/20200409T212949Z-IR-AS197207-sni_blocking-20200409T212950Z_AS197207_5jAUTq66SY2un67orXDtyHEHcaSACm1Fw72j98Ln0ZpisNTGB9-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-09/20200409T213306Z-IR-AS197207-sni_blocking-20200409T213307Z_AS197207_T5UeuvHka5egydT420BAVAxTcd6tvc3AEVpW858bGKgpbCvYus-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T155258Z-IR-AS12880-sni_blocking-20200411T155301Z_AS12880_mJYzWrzE9SToHeNSPStEW22SYOgMBIpzci49cDeXvKr2TtlA9l-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T155418Z-IR-AS12880-sni_blocking-20200411T155420Z_AS12880_BcF5JZLNF6rVW9zqKdxWrovIg8Ef2t9YRqSPDCP4rXiVIOIjJB-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T155625Z-IR-AS197207-sni_blocking-20200411T155627Z_AS197207_VlF35wOyEbESrCD4T0c8KpiKdsexqrdMwH5cqkL9OYLLY3040f-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T155641Z-IR-AS197207-sni_blocking-20200411T155643Z_AS197207_quY22qYKc0kM2WtTv3bM8u3MRBxPSw0I3m9O8PxRuO29uUwmEX-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T161630Z-IR-AS197207-sni_blocking-20200411T161632Z_AS197207_xfPU2JDCDawaiByTqnTqLBka7DUIjjhJtojNezEof4UwIzQR2f-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T161703Z-IR-AS197207-sni_blocking-20200411T161706Z_AS197207_jLQF2HUG41OqgYxKlXSlSHJ3zFjRgjyHI5qcCgk78SWt6llMAN-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T162317Z-IR-AS12880-sni_blocking-20200411T162319Z_AS12880_bWjZSjO1kUPR6QjS6wv4WUcksCfbyvAm4qLfxRjoOqbGJea026-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T162334Z-IR-AS12880-sni_blocking-20200411T162336Z_AS12880_BjvXzehDVigXFu5L5WavaqiQITR9YJbUQOuSh2HDlxN71bCcDM-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T183848Z-IR-AS58224-sni_blocking-20200411T183850Z_AS58224_MRR9XePYRxzwLlUbsjfcGeGtuDI48hQhT7lSsZgpt4NWxSoMEb-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T183909Z-IR-AS58224-sni_blocking-20200411T183911Z_AS58224_7hkLvDfNrmE7j4vdbURyMbUfHgmClUecD9y5H6n4x4SskWud3C-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T184121Z-IR-AS58224-sni_blocking-20200411T184123Z_AS58224_OTuCua7Bi6spBe6KORX6ZunNxEYYdzOzCYQEMgRbueRZorGT7i-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T185137Z-IR-AS58224-sni_blocking-20200411T185140Z_AS58224_FVGaL1kLIbzUW6XNCZ8yH28nMb1iqTMUcNzCka8FHSuwX9ZlaL-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T200716Z-IR-AS58224-sni_blocking-20200411T200719Z_AS58224_7x2GOmyog6TAfKj4YwUcPcDtLVSrMatXissdKbtB9q4WrdfXaV-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-11/20200411T202755Z-IR-AS58224-sni_blocking-20200411T202756Z_AS58224_Ghs4k2OtkOTSnzgeDZ1u5PdTkEd5vE0WggqeMYDhKAXYRCYyHi-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T212514Z-IR-AS58224-sni_blocking-20200412T212514Z_AS58224_Ykytm7DP4EZJBit2PyKxGFFdZqhVosMAFbzKO3jatpzlznTOSC-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T212712Z-IR-AS58224-sni_blocking-20200412T212712Z_AS58224_rjNLA0pHg8cimpYDhgQrO9tTYk4XskYZk0plKMQuO1Ie1BFFD4-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T213900Z-IR-AS58224-sni_blocking-20200412T213901Z_AS58224_6LMgGlOj9TKrURiK4WYjbvNHkYmQoksh1axz5NFMWqVPQYlh5U-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T213949Z-IR-AS58224-sni_blocking-20200412T213952Z_AS58224_FDXS8yliIuvPvyInAMqwl4WFdmoVKVXZnqw8nafttMPLmEx66R-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T214035Z-IR-AS58224-sni_blocking-20200412T214036Z_AS58224_5gh33PX4iS3a0oGxuBu3eIlPWFpvfUmo1qj9wHPaW6dD3yynAe-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T214125Z-IR-AS58224-sni_blocking-20200412T214126Z_AS58224_ZkMsLeGvWKl7LcxbpPh4vhgoZnvSmrQCzV3b1LzGjwAsaRzlPS-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T214202Z-IR-AS58224-sni_blocking-20200412T214202Z_AS58224_AQEaM9KhvbY9pQFq7tA74EA6H3zRYYrqaBWeP6aqQYmqdwbXJX-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T214228Z-IR-AS58224-sni_blocking-20200412T214229Z_AS58224_eoPof8zVmPHyzBsETSjG9TuhYTosI5NWX9sjqHL9OluCkAtHjj-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T214305Z-IR-AS58224-sni_blocking-20200412T214306Z_AS58224_nr0iQBb9rSvQuVOZ8Zi5wXM33atBhZYZbvTlX4zYpORJoU0pGb-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T214337Z-IR-AS58224-sni_blocking-20200412T214338Z_AS58224_tLAPU84tp76uvLmxTU0I8K8pBxQSy8PFzj7dkSlsd1d6MLySzt-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T214416Z-IR-AS58224-sni_blocking-20200412T214417Z_AS58224_msGBQgm1U8U6zayiAFT9W807kOE15M0aJhfYCjRCbg4TuvY5BK-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T220427Z-IR-AS58224-sni_blocking-20200412T220428Z_AS58224_OJnDz5cmkEWZRpZMBUC5fSKhutWoIZMqhv48vsmHUAZJ7Fc63I-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T220500Z-IR-AS58224-sni_blocking-20200412T220501Z_AS58224_67lIBH3KBBK3Hyz8x2sQ2dlSDfrRlkmWCHeRDcDZopo4oAChJ9-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T220647Z-IR-AS58224-sni_blocking-20200412T220648Z_AS58224_946wSYJJcMNyjPnvc3v9h9FgY4s0foibp7PNAYymu3aohFpG1Q-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T220738Z-IR-AS58224-sni_blocking-20200412T220739Z_AS58224_0sQmOMZoOfuQsj3ivRauXTiyOcRpsm2P1NhcbL5h6ch6RmhOU7-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-12/20200412T220837Z-IR-AS58224-sni_blocking-20200412T220838Z_AS58224_XLNg8Rzo9vC4IcLNEl1GzQwBgk4BFe8aL938Fd58DVB83MfSN8-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-13/20200413T221230Z-IR-AS58224-sni_blocking-20200413T221231Z_AS58224_AIpFCfXAJJMXWpo5K4rhdIJwdBwya6rDwjZnZF5I5xrhK7KHEI-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-13/20200413T221313Z-IR-AS58224-sni_blocking-20200413T221314Z_AS58224_BocUcu8jUgvt9wIrWxQvefYmL4nv7mbnKXZIzKu4Isyfn7lQne-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-14/20200414T001704Z-IR-AS58224-sni_blocking-20200414T001705Z_AS58224_VGwqPOLcMHawOtVAZgb4mXIPk3jzIoR41B10YzjENJJZdZ6GF8-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-14/20200414T001803Z-IR-AS58224-sni_blocking-20200414T001804Z_AS58224_X4xaMqPxkDFsWws3FPrvZvqSwU8sGUENhFslcwkzCzAHO62g4e-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-14/20200414T001843Z-IR-AS58224-sni_blocking-20200414T001845Z_AS58224_BunqGYX0DEQdN5WJ8PSmXlevIneUTAIbvG7MXmei39ptkJOEz6-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-14/20200414T002007Z-IR-AS58224-sni_blocking-20200414T002008Z_AS58224_rCfpgFG761SCykiEFAId7F00BFfPgMLEoHQYBuaKwu6DJRv7NU-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-14/20200414T002059Z-IR-AS58224-sni_blocking-20200414T002100Z_AS58224_A99qXwftNCAw7Ef5gKKtO8EIzlyiLqNVYhqkN8URlkh2S4Q5Bt-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-14/20200414T002118Z-IR-AS58224-sni_blocking-20200414T002120Z_AS58224_l5ByP1VNLFoyTw8bbGSDeRdLw4gInXHXhsuG47X2B8qpsGJXEJ-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-14/20200414T002200Z-IR-AS58224-sni_blocking-20200414T002201Z_AS58224_E9WKPRO5SxHr4xQYtYHTJgn0EUZWh9fFEwSjuKzEnalsbiSokB-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-14/20200414T002242Z-IR-AS58224-sni_blocking-20200414T002243Z_AS58224_RguZjYfKLGPDhniGxl6ieylzATgHAWKTuxwrx6thg8xUgwgs1R-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-14/20200414T002327Z-IR-AS58224-sni_blocking-20200414T002328Z_AS58224_57xpy9UMIYTEiy2XLPt6siDWczW54WcaqmdiFKKbhET757XJ3u-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-14/20200414T171522Z-IR-AS197207-sni_blocking-20200414T171525Z_AS197207_WInpPTeSmpXoLFFmYwe1U50qCQiwfNNjgp7NHkPTGSHywswUgu-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-14/20200414T171549Z-IR-AS197207-sni_blocking-20200414T171552Z_AS197207_fkCk0lWu8YykqWNkmTwJygK1xWmZVrOsKz041R0ECABdUOH4vO-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-14/20200414T171645Z-IR-AS197207-sni_blocking-20200414T171647Z_AS197207_7pBD3QWA8OjRLPtP2GqFNb8087iBN4PxKhRWnOE5Dyo1NHzW9d-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-15/20200415T223050Z-IR-AS58224-sni_blocking-20200415T223051Z_AS58224_Ep7YdGtj0712dIT2zf5Sy79HkgidmVTBLLliLH7st0J9YAoAVF-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-17/20200417T094352Z-IR-AS58224-sni_blocking-20200417T094354Z_AS58224_G7Rz1C29sNBCC7dXIB7is35EDmGUMRj3QvoQ0mdghthfmidHZH-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-17/20200417T094432Z-IR-AS58224-sni_blocking-20200417T094433Z_AS58224_883GkOxfIWsW2rnT7zLv5rAm3KaqE8rjaZp7u5ziUNynY32HcP-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-17/20200417T164030Z-IR-AS58224-sni_blocking-20200417T164032Z_AS0_QXckEgSnuXAIEuIORnmUPUW674eZSwUKqqai4gIhUczOJbKfqz-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-17/20200417T165229Z-IR-AS58224-sni_blocking-20200417T165230Z_AS58224_zTP2AzTg3f4ppq9ieiUi7Mizcqil5rJ1XmYY2MXOTuW6VURGRR-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-17/20200417T165348Z-IR-AS58224-sni_blocking-20200417T165350Z_AS58224_j4CMjo0BTkPZWMQJmsn5fkRoha8SFTNiuZq22Hkmfqgg1HQlpV-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-18/20200418T105437Z-IR-AS58224-sni_blocking-20200418T105440Z_AS58224_GguWqwSThCto4VAEZRXIvcYMK7QILkHt9LFAed09mTBLM2LsgX-0.2.0-probe.json .
aws s3 cp --no-sign-request s3://ooni-data/autoclaved/jsonl/2020-04-19/20200419T164132Z-IR-AS58224-sni_blocking-20200419T164134Z_AS58224_dYomtLOMQAa3Szms1jK9S0oxoT2YJsEqHoHaA16UhWoafyhZpN-0.2.0-probe.json .
#!/usr/bin/env python3
import json
import sys
def main():
allasn = ["AS12880", "AS58224", "AS197207"]
skip = ["kernel.org", "www.kernel.org", "example.com", "www.example.com"]
table = {}
for filepath in sys.argv[1:]:
with open(filepath, "rb") as filep:
for line in filep:
data = json.loads(line)
domain = data["input"]
if domain in skip:
continue
if domain not in table:
table[domain] = {}
probe_asn = data["probe_asn"]
if probe_asn not in allasn:
continue
tk = data["test_keys"]
try:
result = tk["result"]
except KeyError:
continue # old measurement w/o result
result = result.split(".")[1]
table[domain].setdefault(probe_asn, {}).setdefault(result, 0)
table[domain][probe_asn][result] += 1
descr = {
"AS12880": "(DCI)",
"AS58224": "(TCI)",
"AS197207": "(MCI)"
}
header = "| DOMAIN"
for asn in allasn:
header += " | " + asn + " " + descr[asn]
header += " |"
print(header)
print("| ------ | ------- | ------- | -------- |")
for domain in sorted(table):
value = "| " + domain
for asn in allasn:
v = table[domain].get(asn, None)
if v is not None:
assert len(v) == 1
v = list(v.keys())[0]
value += " | " + json.dumps(v)
value += " |"
print(value)
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.