createSubmoduleDeps.sh

#!/usr/bin/env bash

set -euo pipefail

git_repo="/home/alex/git/pytorch"
cmd_var="local_extract_cmd_pattern"
sub_cmd_var="local_mkdir_and_extract_cmd_pattern"
thirdparty_dir="third_party"
thirdparty_var="local_pytorchthirdpartydir"
# Repo names for dependencies that should be ignored
repo_ignore=(zstd nccl six enum34 ios-cmake ARM_NEON_2_x86_SSE clang-cindex-python3)
# Subdir of git_repo to ignore
subdir_ignore=("onnx-tensorrt/third_party/onnx" "protobuf/third_party/benchmark" "protobuf/third_party/googletest")

function printSubmoduleDeps(){
    local dir="$1"
    local submodules_dir="$2"
    local extract_cmd="$3"
    local submodules

    mapfile -t submodules < <(cd "$dir" && git submodule status "$submodules_dir")

    for submodule_data in "${submodules[@]}"; do
        hash=$(echo "$submodule_data" | awk '{print $1;}')
        submodule=$(echo "$submodule_data" | awk '{print $2;}')

        if [[ "${hash::1}" == "-" ||  "${hash::1}" == "+" ||  "${hash::1}" == "U" ]]; then
            echo "Submodule $submodule is dirty. Please reset"
            exit 1
        fi

        submodule_dir="$dir/$submodule"
        subfolder=${submodule_dir#*/$thirdparty_dir/}

        if [[ " ${subdir_ignore[*]} " =~ " $subfolder " ]]; then
            continue
        fi

        remote=$(cd "$submodule_dir" && git remote get-url origin)
        remote="${remote%.git}"
        source_url="${remote}/archive"

        if [[ " ${repo_ignore[*]} " =~ " $(basename "$remote") " ]]; then
            continue
        fi

        commit_date=$(cd "$submodule_dir" && git show -s --format='%cd' --date=format:'%Y%m%d')
        filename="$(basename "$remote")-${commit_date}.tar.gz"
        filename="${filename/-git-mirror/}"

        echo "\
    {
        'source_urls': ['$source_url'],
        'download_filename': '${hash}.tar.gz',
        'filename': '$filename',
        'extract_cmd': $extract_cmd % ($thirdparty_var, '$subfolder'),
    },"
        printSubmoduleDeps "$submodule_dir" "." "$cmd_var"
    done
}

printSubmoduleDeps "$git_repo" "$thirdparty_dir" "$cmd_var"

generated.py

    {
        'source_urls': ['https://github.com/Maratyszcza/FP16/archive'],
        'download_filename': 'febbb1c163726b5db24bed55cc9dc42529068997.tar.gz',
        'filename': 'FP16-20181128.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'FP16'),
    },
    {
        'source_urls': ['https://github.com/Maratyszcza/FXdiv/archive'],
        'download_filename': 'b742d1143724d646cd0f914646f1240eacf5bd73.tar.gz',
        'filename': 'FXdiv-20181116.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'FXdiv'),
    },
    {
        'source_urls': ['https://github.com/Maratyszcza/NNPACK/archive'],
        'download_filename': '24b55303f5cf65d75844714513a0d1b1409809bd.tar.gz',
        'filename': 'NNPACK-20191007.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'NNPACK'),
    },
    {
        'source_urls': ['https://github.com/pytorch/QNNPACK/archive'],
        'download_filename': '7d2a4e9931a82adc3814275b6219a03e24e36b4c.tar.gz',
        'filename': 'QNNPACK-20190828.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'QNNPACK'),
    },
    {
        'source_urls': ['https://github.com/google/benchmark/archive'],
        'download_filename': '505be96ab23056580a3a2315abba048f4428b04e.tar.gz',
        'filename': 'benchmark-20180606.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'benchmark'),
    },
    {
        'source_urls': ['https://github.com/pytorch/cpuinfo/archive'],
        'download_filename': '89fe1695edf9ee14c22f815f24bac45577a4f135.tar.gz',
        'filename': 'cpuinfo-20190117.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'cpuinfo'),
    },
    {
        'source_urls': ['https://github.com/NVlabs/cub/archive'],
        'download_filename': '285aeebaa34b0e8a7670867a2e66c1a52d998d6a.tar.gz',
        'filename': 'cub-20170829.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'cub'),
    },
    {
        'source_urls': ['https://github.com/eigenteam/eigen-git-mirror/archive'],
        'download_filename': 'd41dc4dd74acce21fb210e7625d5d135751fa9e5.tar.gz',
        'filename': 'eigen-20190125.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'eigen'),
    },
    {
        'source_urls': ['https://github.com/pytorch/fbgemm/archive'],
        'download_filename': '6aaaa4754fe2a7467c5f0584648ba47482caad7d.tar.gz',
        'filename': 'fbgemm-20191120.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm'),
    },
    {
        'source_urls': ['https://github.com/asmjit/asmjit/archive'],
        'download_filename': '17556b2d4984ab67bdc00e6b5a626efc75d95e07.tar.gz',
        'filename': 'asmjit-20191001.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm/third_party/asmjit'),
    },
    {
        'source_urls': ['https://github.com/pytorch/cpuinfo/archive'],
        'download_filename': 'd5e37adf1406cf899d7d9ec1d317c47506ccb970.tar.gz',
        'filename': 'cpuinfo-20190201.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm/third_party/cpuinfo'),
    },
    {
        'source_urls': ['https://github.com/google/googletest/archive'],
        'download_filename': '0fc5466dbb9e623029b1ada539717d10bd45e99e.tar.gz',
        'filename': 'googletest-20180925.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm/third_party/googletest'),
    },
    {
        'source_urls': ['https://github.com/houseroad/foxi/archive'],
        'download_filename': '97fe555430a857581b9b826ecd955e4f0a3653f0.tar.gz',
        'filename': 'foxi-20191029.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'foxi'),
    },
    {
        'source_urls': ['https://github.com/google/gemmlowp/archive'],
        'download_filename': '3fb5c176c17c765a3492cd2f0321b0dab712f350.tar.gz',
        'filename': 'gemmlowp-20181126.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'gemmlowp/gemmlowp'),
    },
    {
        'source_urls': ['https://github.com/facebookincubator/gloo/archive'],
        'download_filename': '7c541247a6fa49e5938e304ab93b6da661823d0f.tar.gz',
        'filename': 'gloo-20191105.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'gloo'),
    },
    {
        'source_urls': ['https://github.com/google/googletest/archive'],
        'download_filename': '2fe3bd994b3189899d93f1d5a881e725e046fdc2.tar.gz',
        'filename': 'googletest-20180831.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'googletest'),
    },
    {
        'source_urls': ['https://github.com/intel/ideep/archive'],
        'download_filename': '7def110361c05ee18569272e7b4e0b370998c6b1.tar.gz',
        'filename': 'ideep-20191008.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'ideep'),
    },
    {
        'source_urls': ['https://github.com/intel/mkl-dnn/archive'],
        'download_filename': '7d2fd500bc78936d1d648ca713b901012f470dbc.tar.gz',
        'filename': 'mkl-dnn-20190927.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'ideep/mkl-dnn'),
    },
    {
        'source_urls': ['https://github.com/onnx/onnx/archive'],
        'download_filename': 'fea8568cac61a482ed208748fdc0e1a8e47f62f5.tar.gz',
        'filename': 'onnx-20191106.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx'),
    },
    {
        'source_urls': ['https://github.com/google/benchmark/archive'],
        'download_filename': 'e776aa0275e293707b6a0901e0e8d8a8a3679508.tar.gz',
        'filename': 'benchmark-20180525.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx/third_party/benchmark'),
    },
    {
        'source_urls': ['https://github.com/pybind/pybind11/archive'],
        'download_filename': 'a1041190c8b8ff0cd9e2f0752248ad5e3789ea0c.tar.gz',
        'filename': 'pybind11-20170828.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx/third_party/pybind11'),
    },
    {
        'source_urls': ['https://github.com/onnx/onnx-tensorrt/archive'],
        'download_filename': 'c153211418a7c57ce071d9ce2a41f8d1c85a878f.tar.gz',
        'filename': 'onnx-tensorrt-20190916.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx-tensorrt'),
    },
    {
        'source_urls': ['https://github.com/protocolbuffers/protobuf/archive'],
        'download_filename': '48cb18e5c419ddd23d9badcfe4e9df7bde1979b2.tar.gz',
        'filename': 'protobuf-20180727.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'protobuf'),
    },
    {
        'source_urls': ['https://github.com/Maratyszcza/psimd/archive'],
        'download_filename': '90a938f30ba414ada2f4b00674ee9631d7d85e19.tar.gz',
        'filename': 'psimd-20180906.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'psimd'),
    },
    {
        'source_urls': ['https://github.com/Maratyszcza/pthreadpool/archive'],
        'download_filename': '13da0b4c21d17f94150713366420baaf1b5a46f4.tar.gz',
        'filename': 'pthreadpool-20181008.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'pthreadpool'),
    },
    {
        'source_urls': ['https://github.com/pybind/pybind11/archive'],
        'download_filename': '25abf7efba0b2990f5a6dfb0a31bc65c0f2f4d17.tar.gz',
        'filename': 'pybind11-20190204.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'pybind11'),
    },
    {
        'source_urls': ['https://github.com/Maratyszcza/PeachPy/archive'],
        'download_filename': '07d8fde8ac45d7705129475c0f94ed8925b93473.tar.gz',
        'filename': 'PeachPy-20180218.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'python-peachpy'),
    },
    {
        'source_urls': ['https://github.com/shibatch/sleef/archive'],
        'download_filename': '7f523de651585fe25cade462efccca647dcc8d02.tar.gz',
        'filename': 'sleef-20190730.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'sleef'),
    },
    {
        'source_urls': ['https://github.com/01org/tbb/archive'],
        'download_filename': 'a51a90bc609bb73db8ea13841b5cf7aa4344d4a9.tar.gz',
        'filename': 'tbb-20181009.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'tbb'),
    },

orig.py

    {
        'source_urls': ['https://github.com/Maratyszcza/FP16/archive'],
        'download_filename': 'febbb1c163726b5db24bed55cc9dc42529068997.tar.gz',
        'filename': 'FP16-20181128.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'FP16'),
    },
    {
        'source_urls': ['https://github.com/Maratyszcza/FXdiv/archive'],
        'download_filename': 'b742d1143724d646cd0f914646f1240eacf5bd73.tar.gz',
        'filename': 'FXdiv-20181016.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'FXdiv'),
    },
    {
        'source_urls': ['https://github.com/Maratyszcza/NNPACK/archive'],
        'download_filename': '24b55303f5cf65d75844714513a0d1b1409809bd.tar.gz',
        'filename': 'NNPACK-20191008.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'NNPACK'),
    },
    {
        'source_urls': ['https://github.com/pytorch/QNNPACK/archive'],
        'download_filename': '7d2a4e9931a82adc3814275b6219a03e24e36b4c.tar.gz',
        'filename': 'QNNPACK-20190828.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'QNNPACK'),
    },
    {
        'source_urls': ['https://github.com/google/benchmark/archive'],
        'download_filename': '505be96ab23056580a3a2315abba048f4428b04e.tar.gz',
        'filename': 'benchmark-20180606.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'benchmark'),
    },
    {
        'source_urls': ['https://github.com/pytorch/cpuinfo/archive'],
        'download_filename': '89fe1695edf9ee14c22f815f24bac45577a4f135.tar.gz',
        'filename': 'cpuinfo-20190117.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'cpuinfo'),
    },
    {
        'source_urls': ['https://github.com/NVlabs/cub/archive'],
        'download_filename': '285aeebaa34b0e8a7670867a2e66c1a52d998d6a.tar.gz',
        'filename': 'cub-20170829.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'cub'),
    },
    {
        'source_urls': ['https://github.com/eigenteam/eigen-git-mirror/archive'],
        'download_filename': 'd41dc4dd74acce21fb210e7625d5d135751fa9e5.tar.gz',
        'filename': 'eigen-20190126.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'eigen'),
    },
    {
        'source_urls': ['https://github.com/pytorch/fbgemm/archive'],
        'download_filename': '6aaaa4754fe2a7467c5f0584648ba47482caad7d.tar.gz',
        'filename': 'fbgemm-20191120.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm'),
    },
    {
        'source_urls': ['https://github.com/asmjit/asmjit/archive'],
        'download_filename': '17556b2d4984ab67bdc00e6b5a626efc75d95e07.tar.gz',
        'filename': 'asmjit-20190930.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm/third_party/asmjit'),
    },
    {
        'source_urls': ['https://github.com/pytorch/cpuinfo/archive'],
        'download_filename': 'd5e37adf1406cf899d7d9ec1d317c47506ccb970.tar.gz',
        'filename': 'cpuinfo-20190201.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm/third_party/cpuinfo'),
    },
    {
        'source_urls': ['https://github.com/google/googletest/archive'],
        'download_filename': '0fc5466dbb9e623029b1ada539717d10bd45e99e.tar.gz',
        'filename': 'googletest-20180920.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'fbgemm/third_party/googletest'),
    },
    {
        'source_urls': ['https://github.com/houseroad/foxi/archive'],
        'download_filename': '97fe555430a857581b9b826ecd955e4f0a3653f0.tar.gz',
        'filename': 'foxi-20191029.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'foxi'),
    },
    {
        'source_urls': ['https://github.com/google/gemmlowp/archive'],
        'download_filename': '3fb5c176c17c765a3492cd2f0321b0dab712f350.tar.gz',
        'filename': 'gemmlowp-20181127.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'gemmlowp/gemmlowp'),
    },
    {
        'source_urls': ['https://github.com/facebookincubator/gloo/archive'],
        'download_filename': '7c541247a6fa49e5938e304ab93b6da661823d0f.tar.gz',
        'filename': 'gloo-20191105.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'gloo'),
    },
    {
        'source_urls': ['https://github.com/google/googletest/archive'],
        'download_filename': '2fe3bd994b3189899d93f1d5a881e725e046fdc2.tar.gz',
        'filename': 'googletest-20180831.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'googletest'),
    },
    {
        'source_urls': ['https://github.com/intel/ideep/archive'],
        'download_filename': '7def110361c05ee18569272e7b4e0b370998c6b1.tar.gz',
        'filename': 'ideep-20191008.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'ideep'),
    },
    {
        'source_urls': ['https://github.com/intel/mkl-dnn/archive'],
        'download_filename': '7d2fd500bc78936d1d648ca713b901012f470dbc.tar.gz',
        'filename': 'mkl-dnn-20190928.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'ideep/mkl-dnn'),
    },
    {
        'source_urls': ['https://github.com/onnx/onnx/archive'],
        'download_filename': 'fea8568cac61a482ed208748fdc0e1a8e47f62f5.tar.gz',
        'filename': 'onnx-20191107.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx'),
    },
    {
        'source_urls': ['https://github.com/google/benchmark/archive'],
        'download_filename': 'e776aa0275e293707b6a0901e0e8d8a8a3679508.tar.gz',
        'filename': 'benchmark-20180525.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx/third_party/benchmark'),
    },
    {
        'source_urls': ['https://github.com/pybind/pybind11/archive'],
        'download_filename': 'a1041190c8b8ff0cd9e2f0752248ad5e3789ea0c.tar.gz',
        'filename': 'pybind11-20170828.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx/third_party/pybind11'),
    },
    {
        'source_urls': ['https://github.com/onnx/onnx-tensorrt/archive'],
        'download_filename': 'c153211418a7c57ce071d9ce2a41f8d1c85a878f.tar.gz',
        'filename': 'onnx-tensorrt-20190916.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'onnx-tensorrt'),
    },
    {
        'source_urls': ['https://github.com/google/protobuf/archive'],
        'download_filename': '48cb18e5c419ddd23d9badcfe4e9df7bde1979b2.tar.gz',
        'filename': 'protobuf-20180727.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'protobuf'),
    },
    {
        'source_urls': ['https://github.com/Maratyszcza/psimd/archive'],
        'download_filename': '90a938f30ba414ada2f4b00674ee9631d7d85e19.tar.gz',
        'filename': 'psimd-20180906.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'psimd'),
    },
    {
        'source_urls': ['https://github.com/Maratyszcza/pthreadpool/archive'],
        'download_filename': '13da0b4c21d17f94150713366420baaf1b5a46f4.tar.gz',
        'filename': 'pthreadpool-20181008.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'pthreadpool'),
    },
    {
        'source_urls': ['https://github.com/pybind/pybind11/archive'],
        'download_filename': '25abf7efba0b2990f5a6dfb0a31bc65c0f2f4d17.tar.gz',
        'filename': 'pybind11-20190204.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'pybind11'),
    },
    {
        'source_urls': ['https://github.com/wjakob/clang-cindex-python3/archive'],
        'download_filename': '6a00cbc4a9b8e68b71caf7f774b3f9c753ae84d5.tar.gz',
        'filename': 'clang-cindex-python3-20170330.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'pybind11/tools/clang'),
    },
    {
        'source_urls': ['https://github.com/Maratyszcza/PeachPy/archive'],
        'download_filename': '07d8fde8ac45d7705129475c0f94ed8925b93473.tar.gz',
        'filename': 'PeachPy-20180219.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'python-peachpy'),
    },
    {
        'source_urls': ['https://github.com/shibatch/sleef/archive'],
        'download_filename': '7f523de651585fe25cade462efccca647dcc8d02.tar.gz',
        'filename': 'sleef-20190730.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'sleef'),
    },
    {
        'source_urls': ['https://github.com/intel/tbb/archive'],
        'download_filename': 'a51a90bc609bb73db8ea13841b5cf7aa4344d4a9.tar.gz',
        'filename': 'tbb-20181009.tar.gz',
        'extract_cmd': local_extract_cmd_pattern % (local_pytorchthirdpartydir, 'tbb'),
    },