gh-check: speed test to known GitHub IPs

gh-check

#!/usr/bin/python3

import asyncio
import time
import socket
import argparse

import aiohttp

class MyConnector(aiohttp.TCPConnector):
  def __init__(self, ip):
    self.__ip = ip
    super().__init__()

  async def _resolve_host(
    self, host: str, port: int,
    traces: None = None,
  ):
    return [{
      'hostname': host, 'host': self.__ip, 'port': port,
      'family': self._family, 'proto': 0, 'flags': 0,
    }]

async def test_domain(domain, ip, proto):
  if proto == 'http':
    return await test_domain_http(domain, ip)
  elif proto == 'ssh':
    return await test_domain_ssh(domain, ip)
  else:
    raise ValueError('unknown proto', proto)

async def test_domain_ssh(domain, ip):
  st = time.time()
  r, _w = await asyncio.open_connection(ip, 22)
  await r.read(1)
  return time.time() - st

async def test_domain_http(domain, ip):
  url = 'https://github.com/'
  st = time.time()
  async with aiohttp.ClientSession(
    connector = MyConnector(ip),
    timeout = aiohttp.ClientTimeout(total=10),
  ) as s:
    r = await s.get(url)
    _ = await r.text()

  return time.time() - st

async def producer(q, proto):
  items = await get_items(proto)
  for item in items:
    await q.put(item)

  await q.put(None)

async def printer(q):
  while True:
    try:
      item = await q.get()
    except asyncio.CancelledError:
      break

    if isinstance(item[1], Exception):
      (domain, ip, proto), e = item
      print(f'{domain:21} {ip:15} {proto:4} {e!r}')
    else:
      (domain, ip, proto), t = item
      print(f'{domain:21} {ip:15} {proto:4} {t:6.2f}')

async def fastest_finder(q):
  fastest_ip, latency = None, 1000

  while True:
    try:
      item = await q.get()
    except asyncio.CancelledError:
      return fastest_ip

    if not isinstance(item[1], Exception):
      (_, ip, _), t = item
      if t < latency:
        latency = t
        fastest_ip = ip

async def worker(q, ret_q):
  while True:
    item = await q.get()
    if item is None:
      await q.put(None)
      break

    try:
      t = await test_domain(*item)
    except Exception as e:
      await ret_q.put((item, e))
    else:
      await ret_q.put((item, t))

async def main(proto):
  q = asyncio.Queue()
  ret_q = asyncio.Queue()

  futures = [worker(q, ret_q) for _ in range(40)]
  producer_fu = asyncio.ensure_future(producer(q, proto))
  printer_fu = asyncio.ensure_future(printer(ret_q))

  await asyncio.gather(*futures)
  printer_fu.cancel()
  await producer_fu
  await printer_fu

async def update_hosts():
  import os, sys, subprocess

  if os.geteuid() != 0:
    sys.exit('not root?')

  q = asyncio.Queue()
  ret_q = asyncio.Queue()

  futures = [worker(q, ret_q) for _ in range(40)]
  producer_fu = asyncio.ensure_future(
    producer(q, ['http']))
  finder_fu = asyncio.ensure_future(
    fastest_finder(ret_q))

  await asyncio.gather(*futures)
  finder_fu.cancel()
  await producer_fu
  ip = await finder_fu

  if ip is not None:
    cmd = ['sed', '-Ei', rf'/^[0-9.]+[[:space:]]+(gist\.)?github\.com\>/s/[^[:space:]]+/{ip}/', '/etc/hosts']
    subprocess.check_call(cmd)

async def resolve(domain):
  loop = asyncio.get_current_loop()
  addrinfo = await loop.getaddrinfo(
    domain, None,
    family=socket.AF_INET,
    proto=socket.IPPROTO_TCP,
  )
  ips = [x[-1][0] for x in addrinfo]
  return domain, ips

async def get_items(proto):
  items = [
    ('140.82.112.3',    'Ashburn'),
    ('140.82.112.4',    'Ashburn'),
    ('140.82.113.3',    'Ashburn'),
    ('140.82.113.4',    'Ashburn'),
    ('140.82.114.3',    'Ashburn'),
    ('140.82.114.4',    'Ashburn'),
    ('140.82.116.3',    'Seattle'),
    ('140.82.116.4',    'Seattle'),
    ('140.82.121.3',    'Frankfurt'),
    ('140.82.121.4',    'Frankfurt'),
    ('20.200.245.247',  'Seoul'),
    ('20.201.28.151',   'Sao Paulo'),
    ('20.205.243.166',  'Singapore'),
    ('20.207.73.82',    'Pune'),
    ('20.233.83.145',   'Dubai'),
    ('20.26.156.215',   'London'),
    ('20.27.177.113',   'Tokyo'),
    ('20.29.134.23',    'Quincy, WA'),
    ('20.87.245.0',     'Johannesburg'),
    ('4.208.26.197',    'Dublin'),
    ('4.237.22.38',     'Sydney'),
  ]

  return [(x[1], x[0], y) for x in items for y in proto]

if __name__ == '__main__':
  import logging
  logging.getLogger().addHandler(logging.NullHandler())

  parser = argparse.ArgumentParser(
    description='GitHub IP 访问速度测试')
  parser.add_argument('proto', nargs='*',
                      default=['http', 'ssh'],
                      help='测试指定协议')
  parser.add_argument('--hosts',
                      action='store_true',
                      help='更新 /etc/hosts')
  args = parser.parse_args()

  if args.hosts:
    main_fu = update_hosts()
  else:
    main_fu = main(args.proto)

  try:
    asyncio.run(main_fu)
  except KeyboardInterrupt:
    pass

我用 perl 写了一个类似功能的脚本

#!/bin/perl

use warnings;
use strict;

use Time::HiRes qw(gettimeofday);
use Net::Ping;
use threads;
use threads::shared;
use HTTP::Tiny;
use IO::Socket;
use LWP::Simple;
use JSON::PP;
use Class::Struct;
struct(GitHub => {
    web => '@',
    git => '@',
});

# 从 github 的 api 接口获取所有的 github ip 地址
# github api 参考文档地址: https://docs.github.com/cn/rest/reference/meta
# curl -H "Accept: application/vnd.github.v3+json"  https://api.github.com/meta
sub github_ip{
    my $content = get("https://api.github.com/meta");
    if($content) {
        my @web_ip;
        my @git_ip;
        my $response = decode_json $content;
        my @web      = @{$response->{web}};
        my @git      = @{$response->{git}};
        foreach my $ip_with_mask (@web) {
            my ($ip,$mask) = split /\//, $ip_with_mask;
            push @web_ip, $ip;
        }
        foreach my $ip_with_mask (@git) {
            my ($ip,$mask) = split /\//, $ip_with_mask;
            push @git_ip, $ip;
        }
        return GitHub->new(web=>\@web_ip,git => \@git_ip);
    }else {
        print "network failure\n";
        exit 1;
    }
}
print "waiting https://api.github.com/meta ...";
sub check_http {
    my $ip = $_[0];
    my ( $start_sec, $start_mcsecond ) = gettimeofday();
    my $response                       = HTTP::Tiny->new->get("https://$ip");
    $_                                 = $response->{content};
    my ( $end_sec, $end_mcsecond )     = gettimeofday();
    return sprintf("%2.2f",(( $end_sec - $start_sec ) + ( $end_mcsecond - $start_mcsecond ) / ( 1000 * 1000 )));
}


# 检查 ssh 延时, 尝试链接服务器 22 端口，并且读取一个字节
# 返回 2 个值，($ok,$time), $ok 表示是否链接成功，$time 表示链接耗时
sub check_ssh {
    my $sock = IO::Socket::INET->new(
        Proto    => "tcp",
        PeerAddr => $_[0],
        PeerPort => 22,
    );
    my ( $ssh_start_sec, $ssh_start_mcsecond ) = gettimeofday();
    my $len                                    = 1;
    my $buf;
    if(!defined($sock)){
        return "TIMEOUT";
    }
    $sock->sysread($buf,$len);
    my ($ssh_end_sec, $ssh_end_mcsecond ) = gettimeofday();
    my $ssh_time                          = ($ssh_end_sec - $ssh_start_sec) + ($ssh_end_mcsecond - $ssh_start_mcsecond) / (1000*1000);
    return sprintf("%2.2fs",$ssh_time);
}

my $github         = github_ip();
print("done\n");
my @web_ip         = @{$github->web};
my @git_ip         = @{$github->git};
my $count :shared;
$count             = $#web_ip + $#git_ip;
foreach my $ip (@web_ip) {
    threads->new(sub{
        my $host          = $_[0];
        my $http_duration = check_http($host);
        printf "%-20s%-10s%-10s\n",$host,$http_duration,"HTTPS";
        {
            lock($count);
            $count = $count -1
        }
    },$ip)->detach();
}
foreach my $ip (@git_ip) {
    threads->new(sub{
        my $host         = $_[0];
        my $ssh_duration = check_ssh($host);
        printf "%-20s%-10s%-10s\n",$host,$ssh_duration, "SSH";
        {
            lock($count);
            $count = $count -1
        }
    },$ip)->detach();
}
while(1) {
    unless($count) {
        exit(0);
    }
}
```shell
hellojukay@ms7a36 src (master) $ perl github_check.pl
waiting https://api.github.com/meta ...done
13.114.40.48        0.49s     SSH
52.192.72.89        0.49s     SSH
52.74.223.119       0.55s     SSH
54.238.117.237      0.48s     SSH
54.168.17.15        0.49s     SSH
18.181.13.223       0.53s     SSH
13.229.188.59       0.92s     SSH
18.140.96.234       0.58s     SSH
52.64.108.95        0.68s     SSH
18.141.90.153       0.56s     SSH
18.228.67.229       0.59s     SSH
18.231.5.6          0.60s     SSH
3.6.106.81          0.66s     SSH
3.106.158.203       0.64s     SSH
13.250.177.223      1.40s     SSH
13.234.176.102      1.35s     SSH
15.164.81.167       0.55s     SSH
18.231.104.233      0.76s     SSH
18.228.167.86       0.72s     SSH
52.78.231.108       1.65s     SSH
185.199.108.0       2.50      HTTPS
3.7.2.84            1.39s     SSH
18.138.202.180      0.67s     SSH
18.228.52.138       0.64s     SSH
13.125.114.27       1.73s     SSH
52.63.152.235       0.66s     SSH
54.233.131.104      0.60s     SSH
52.69.186.44        1.47s     SSH
13.236.229.21       1.32s     SSH
13.237.44.5         1.44s     SSH
13.234.210.38       1.51s     SSH
3.34.26.58          2.69s     SSH
13.236.229.21       5.27      HTTPS
13.229.188.59       6.92      HTTPS
13.114.40.48        7.45      HTTPS
15.164.81.167       9.48      HTTPS
18.231.5.6          9.30      HTTPS
52.64.108.95        10.61     HTTPS
18.228.67.229       10.96     HTTPS

哇，好久没见过 Perl 脚本了呢～你是解析 meta 的内容的，不过你漏掉了 github 自己的 IP（mask 不为 /32 的那些）。

嗯嗯，是的，谢谢依云指正。