Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Mirror a conda repository
#!/usr/bin/env python
from __future__ import print_function
import json
import os
import sys
def check(path, checksum=False):
if checksum:
from conda.utils import md5_file
missing = set()
bad = set()
with open(os.path.join(path, 'repodata.json')) as f:
repodata = json.load(f)
for fname, info in repodata['packages'].iteritems():
fpath = os.path.join(path, fname)
try:
size = os.path.getsize(fpath)
except OSError:
missing.add(fname)
else:
if size != int(info['size']):
bad.add(fname)
elif checksum:
md5 = md5_file(fpath)
if md5 != info['md5']:
bad.add(fname)
return missing, bad
def main():
import argparse
parser = argparse.ArgumentParser()
g = parser.add_mutually_exclusive_group()
g.add_argument('--checksum', action='store_true', default=False)
g.add_argument('--no-checksum', action='store_false', dest='checksum')
g = parser.add_mutually_exclusive_group()
g.add_argument('--delete-bad', action='store_true', default=False)
g.add_argument('--no-delete-bad', action='store_false', dest='delete_bad')
parser.add_argument('dir')
args = parser.parse_args()
missing, bad = check(args.dir, checksum=args.checksum)
s = ", deleting" if args.delete_bad else ""
if bad:
print("These files are bad{}:".format(s))
for fn in bad:
print("\t{}".format(fn))
if args.delete_bad:
os.remove(os.path.join(args.dir, fn))
if missing:
print("Missing these packages:")
for fname in sorted(missing):
print("\t{}".format(fname))
sys.exit(bool(missing) or bool(bad))
if __name__ == '__main__':
main()
#!/bin/bash
url="$(echo $1 | sed -E 's|/$||')"
path="${2:-$(echo $url | sed -E 's|^https?://||')}"
cut_dirs=$(echo "$url" | sed -E 's|^https?://()/?$|\1/|' | tr -Cd / | wc -c)
wget -N -P "$path" "$url"/repodata.json{,.bz2}
wget -r -nc -nH -P "$path" --cut-dirs $cut_dirs "$url/"
$(dirname $0)/check_repodata.py "$path"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.