Get Rust crates' links and MD5 checksums for Slackware's .info files

get-crates.py

#!/usr/bin/env python3

# Funny enough, this script does everything but actually get crates :D

import json
import requests
import tarfile
import re
from sys import exit, stderr
from hashlib import md5, sha256
from os import mkdir, walk, remove, getcwd, makedirs
from os.path import isfile, join
from glob import glob

def getNeccessaryCrates(workdir):
    "We examine Cargo.lock for a complete list of dependencies with their locked versions. Entries with no checksums are not meant to be downloaded"
    # Compile the regex patterns
    name_pattern = re.compile(r'^\s*name = "([^"]+)"')
    version_pattern = re.compile(r'^\s*version = "([^"]+)"')
    checksum_pattern = re.compile(r'^\s*checksum = "([0-9a-f]{64})"')

    # Initialize a list to store crate names
    crates_with_versions = []

    # Open and read the Cargo.lock file
    with open(workdir + '/Cargo.lock', 'r') as file:
        current_crate_name = None
        for line in file:
            # Check for crate name
            name_match = name_pattern.match(line)
            if name_match:
                current_crate_name = name_match.group(1)

            # Check for crate version
            version_match = version_pattern.match(line)
            if version_match and current_crate_name:
                current_version = version_match.group(1)

            # Check for checksum
            checksum_match = checksum_pattern.match(line)

            # We are only interested in crates with specified checksums
            if checksum_match and current_crate_name:
                checksum = checksum_match.group(1)
                crates_with_versions.append((current_crate_name, current_version))
                # Reset variables after using them
                current_crate_name = None
                current_version = None
    return crates_with_versions

def genLinks(crates_with_versions):
    links = []
    for crate_name, crate_version in crates_with_versions:
        #links.append(f"https://crates.io/api/v1/crates/{crate_name}/{crate_version}/download")
        # Using direct links is much faster than waiting for API to return one
        links.append(f"https://static.crates.io/crates/{crate_name}/{crate_name}-{crate_version}.crate")
    return links

def getChecksums(links):
    "Download crates and get their hashes. Crates are downloaded into memory and are not saved"
    hashes = []
    for link in links:
        hashes.append(md5(requests.get(link).content).hexdigest())
    return hashes

def createVendor(vendordir, cratedir):
    "We extract each crate, create an empty .cargo-ok file, calculate sha256 sum for every file inside the crate and crate file itself and put it into .cargo-checksum.json"
    crate_path = ""
    crate_files, crate_checksums= [], []
    makedirs(vendordir, exist_ok=True)
    for crate in glob(f"{cratedir}/*.crate"):
        with tarfile.open(crate, 'r:*') as archive:
            archive.extractall(path=vendordir, filter='data')
        crate_path = f"{vendordir}/{crate[crate.rfind('/')+1:].replace('.crate', '')}"
        open(f"{crate_path}/.cargo-ok", "a").close()
        for root, dirs, files in walk(f"{crate_path}"):
            crate_files.extend(join(root, name) for name in files)
        for file in crate_files:
            with open(file, "rb") as opened_file:
                crate_checksums.append(sha256(opened_file.read()).hexdigest())
        with open(f"{crate_path}/.cargo-checksum.json", "w") as crate_json:
            with open(crate, 'rb') as crate_file:
                json.dump({"files": dict(zip([file.replace(f"{crate_path}/", "") for file in crate_files], crate_checksums)), "package": sha256(crate_file.read()).hexdigest()}, crate_json)
        crate_files, crate_checksums= [], []


if __name__ == "__main__":
    from shutil import rmtree
    from tempfile import mkdtemp
    import argparse

    parser = argparse.ArgumentParser(
                        prog="get-crates.py",
                        description="This is a helper script for packaging Rust programs using SlackBuilds. It can generate links to crates, their MD5 hashes, fill a template .info file with crates' links and checksums and vendor pre-downloaded crates for offline building with Cargo"
                        )
    parser.add_argument("-i", "--info",      help=".info file template to use. A template is just an .info file without crates' information")
    parser.add_argument("-t", "--tarball",   help="Tarball with a Rust program to generate crates info for")
    parser.add_argument("-d", "--directory", help="Directory of a Rust program to generate crates info for")
    parser.add_argument("-a", "--action",    help="Tell the script what to do. Possible values are: links, md5, info and vendor", default="info")
    parser.add_argument("-s", "--stdout",    help="Display the results to stdout", action="store_true")
    parser.add_argument("-o", "--output",    help="Output to a file")
    parser.add_argument("-c", "--crates",    help="Path to downloaded .crate files to vendor")
    args = parser.parse_args()

    if args.tarball and args.directory:
        print("You have to specify either a tarball or a directory")
        exit(1)
    if args.action not in ["links", "md5", "info", "vendor"]:
        print("Invalid action. Possible values are: links, md5, info and vendor")
        exit(1)
    if args.action == "info" and not args.info:
        print("Please specify .info file template")
        exit(1)
    if args.action == "vendor" and not args.crates:
        print("You must specify a directory with downloaded crates to vendor")
        exit(1)

    # Initialize some variables
    # Look for Cargo.toml in current directory by default
    workdir = getcwd()
    links, hashes = [], []
    template, download, md5sum = "", "", ""
    i, j = 0, 0

    # Set a directory with Cargo.toml
    if args.tarball:
        tmpdir = mkdtemp()
        with tarfile.open(args.tarball, 'r:*') as archive:
            archive.extractall(path=tmpdir, filter='data')
            workdir = tmpdir + "/" + archive.getnames()[0]
    elif args.directory:
        workdir = args.directory

    links = genLinks(getNeccessaryCrates(workdir))

    if args.action == "links":
        if args.stdout:
            for link in links:
                print(link)
        if args.output:
            if isfile(args.output):
                remove(args.output)
            with open(args.output, "w") as output:
                for link in links:
                    output.write(link + "\n")

    if args.action == "md5":
        hashes = getChecksums(links)
        if args.stdout:
            for link, hash in zip(links, hashes):
                print(link, hash)
        if args.output:
            if isfile(args.output):
                remove(args.output)
            with open(args.output, "w") as output:
                for link, hash in zip(links, hashes):
                    output.write(link + " " + hash + "\n")

    if args.action == "info":
        with open(args.info, 'r') as template_file:
            for line in template_file.readlines():
                if line.startswith("DOWNLOAD="):
                    download = f"DOWNLOAD=\"{line.strip()[10:][:-1]} \\\n" # it's a backslash and a newline character at the end
                    i = len(links)
                    for link in links:
                        i -= 1
                        if i != 0 or not line.strip().endswith('"'):
                            download += f"          {link} \\\n"
                        else:
                            download += f"          {link}\"\n"
                    template += download
                elif line.startswith("MD5SUM="):
                    md5sum = f"MD5SUM=\"{line.strip()[8:][:-1]} \\\n"
                    hashes = getChecksums(links)
                    j = len(hashes)
                    for hash in hashes:
                        j -= 1
                        if j != 0 or not line.strip().endswith('"'):
                            md5sum += f"        {hash} \\\n"
                        else:
                            md5sum += f"        {hash}\"\n"
                    template += md5sum
                else:
                    template += line
        if args.stdout:
            print(template)
        if args.output:
            if isfile(args.output):
                remove(args.output)
            with open(args.output, "w") as output:
                output.write(template)

    if args.action == "vendor":
        createVendor(f"{workdir}/vendor", args.crates)

    # Don't forget to remove the temporary directory we unpacked tarball into
    if args.tarball and args.action != "vendor":
        rmtree(tmpdir)
    elif args.tarball and args.action == "vendor":
        print(f"Tempdir {tmpdir} is left behind.", file=stderr)

    exit(0)