GitHub LFS Scan

readme.md

GitHub LFS Scan

Check for the existence of a list of objects in git LFS.

Requirements:

• Requests python library
• a GitHub personal access token in ($GITHUB_PAT)
• a list of objects to look for

The format for lfs_objects.txt looks like the following:

repo: shreve/test-repo
deca46927e2c1e1544dc6f0a206ae9b2c96257bfed4292a7cf7c7f6e023b8a6e 54460700
63e21a001bb4926d83bd368f25e44341e00a03af89b7ecc7ccfc13bb835a288d 13870000
repo: shreve/another-test-repo
55cfd6746fdcc1cb5a29350c5a39c742ae854adbe335a009598f35ae9bcf156e 157050855
dbef09960b9dd4392f144a05562af3639d89e9ce6f167afe928105cbeb76d04b 157811280
2552813c57acf6f65a9190949dd8768c8fc0303b00fe67ac26197c3b228bc4ae 973910290

collect_oids.sh

#!/usr/bin/env bash

# Collect the list of object IDs and their sizes from a single repo
git lfs ls-files --all -d | awk 'NR % 7 == 2 {size=$2} NR % 7 == 5 {oid=$3 } NR % 7 == 0 {print oid " " size}' > files.txt

lfs_scan.py

"""Scan LFS to see if all the objects exist"""

import os
import requests

LFS_URL = "https://github.com/{repo}.git/info/lfs/objects/batch"

DOWNLOAD_REQ = {
    "operation": "download",
    "transfers": ["basic"],
    "objects": [],
}

PAT = os.environ.get("GITHUB_PAT")


def check_repo(repo, objects):
    print("Checking repo", repo, "with", len(objects), "objects")
    request = DOWNLOAD_REQ.copy()
    request["objects"] = objects
    response = requests.post(
        LFS_URL.format(repo=repo),
        json=request,
        headers={
            "Accept": "application/vnd.git-lfs+json",
            "Content-Type": "application/vnd.git-lfs+json",
            "Authorization": f"Bearer {PAT}",
        },
    )

    response.raise_for_status()
    ok = True
    for obj in response.json()["objects"]:
        if obj.get("error"):
            print(obj["error"])
            ok = False

    if ok:
        print(f"{repo} OK\n")
    else:
        print(f"{repo} FAILED\n")


def main():
    """Main function"""

    with open("lfs_objects.txt", "r") as lfs_file:

        repo = None
        objects = []

        for line in lfs_file:
            obj = line.split(" ")

            if obj[0] == "repo:":
                if repo is not None:
                    check_repo(repo, objects)
                repo = obj[1].strip()
                objects = []

            else:
                objects.append({"oid": obj[0], "size": int(obj[1])})


if __name__ == "__main__":
    main()