0-summary.txt

Some basic info about people set their shell's HISTSIZE variable

HISTSIZE stats
count                  473.00
mean        21141872008171.58
std        459800479621288.31
min                     -1.00
25%                   1000.00
50%                  10000.00
75%                  65000.00
max      10000000000000000.00
Name: HISTSIZE, dtype: object

HISTSIZE=-1
count                  12/473
percent    2.536997885835095%

HISTSIZE stats with -1's removed
count                  461.00
mean        21692202732896.23
std        465746422178186.00
min                      0.00
25%                   1000.00
50%                  10000.00
75%                 100000.00
max      10000000000000000.00
Name: HISTSIZE, dtype: object

crawl-github-histsize.py

#!/usr/bin/env python3
"""
Some terrible code to search for "language:shell HISTSIZE=" on GitHub
Copyright 2024 Tyler Cipriani <tyler@tylercipriani.com>
License GPL-3.0+
"""
import base64
import os
import requests
import re
import sys
import time
import datetime


SESSION = requests.Session()
GITHUB_TOKEN = os.environ['GITHUB_TOKEN']


def search_code_request(url: str) -> requests.Response:
    headers = {
        'Accept': 'application/vnd.github+json',
        'Authorization': f'Bearer {GITHUB_TOKEN}',
    }
    response = SESSION.get(url, headers=headers)
    return response.headers, response.json()


def search_code(query: str) -> dict:
    next_url = 'https://api.github.com/search/code?q=' + query
    while True:
        headers, response = search_code_request(next_url)
        link = headers.get('Link')
        if 'items' not in response:
            if headers.get('X-RateLimit-Remaining') == '0':
                current_time = datetime.datetime.now().strftime('%s')
                sleep_time = int(headers['X-RateLimit-Reset']) - int(current_time) + 10
                if sleep_time < 0:
                    sleep_time = 60
                print('Rate limit exceeded', file=sys.stderr)
                print(f'Wait for {sleep_time} seconds', file=sys.stderr)
                time.sleep(sleep_time)
                continue
            else:
                break
        next_url = None
        for item in response['items']:
            yield item
        if link:
            for part in link.split(','):
                url, rel = part.split(';')
                url = url.strip().strip('<>')
                rel = rel.strip()
                if rel == 'rel="next"' and url:
                    next_url = url
                    break
        if not next_url:
            break


def download(url: str) -> str:
    headers = {
        'Accept': 'application/vnd.github+json'
    }
    response = SESSION.get(url, headers=headers)
    return base64.b64decode(response.json()['content']).decode('utf-8')


def main():
    query = 'HISTSIZE=+language:Shell'
    regex = re.compile(r'\bHISTSIZE=(?P<value>-?\d+)')
    with open('histsize.csv', 'w') as f:
        f.write('repository,path,HISTSIZE\n')
        print('repository,path,HISTSIZE')
        for item in search_code(query):
            code = download(item['git_url'])
            for line in code.split('\n'):
                if 'HISTSIZE' not in line:
                    continue
                match = regex.match(line)
                if match:
                    print(f"{item['repository']['full_name']},{item['path']},{match.group('value')}")
                    f.write(f"{item['repository']['full_name']},{item['path']},{match.group('value')}\n")
                    break

if __name__ == '__main__':
    main()

histsize.csv