Skip to content

Instantly share code, notes, and snippets.

Created November 2, 2014 16:19
Show Gist options
  • Save alexwlchan/10e1e24ecd354edc5639 to your computer and use it in GitHub Desktop.
Save alexwlchan/10e1e24ecd354edc5639 to your computer and use it in GitHub Desktop.
Unpack sets and ranges expressed in a string, using cURL syntax for multiple URLs
"""The cURL command line tool allows you to specify multiple URLs in a
single string. For example:
http://site.{one,two,three}.com[1-100].txt[001-100].txt (with leading zeros)[a-z].txt
This module can take a string in this format, and return the list of strings
that it specifies. The exact syntax can be seen in the examples on the cURL man
page [1]. It's rough around the edges, but should work fine with correctly
formatted strings.
To invoke, use the parse_string function on a string:
>>> urls = curlparser.parse_string("http://site.{one,two,three}.com")
>>> files = curlparser.parse_string("[1-100].txt")
or supply the string as a command-line argument (wrap in quotes to avoid the
string being mangled by the shell):
$ python "[a-z:2].txt"
For other notes, see the associated blog post. [2]
Created by Alex Chan ( in November 2014.
import itertools
import re
# These regexes match the different set/range operators. They have () as they
# need to capture the content when we split the string.
ITEMISED_SET = re.compile(r"({[a-zA-Z0-9,]{1,}})")
LOWERCASE_RANGE = re.compile(r"(\[[a-z]{1}\-[a-z]{1}(?::[0-9-]{1})?\])")
UPPERCASE_RANGE = re.compile(r"(\[[A-Z]{1}\-[A-Z]{1}(?::[0-9-]{1})?\])")
NUMERIC_RANGE = re.compile(r"(\[[0-9]{1,}\-[0-9]{1,}(?::-?[0-9]{1,})?\])")
# Utility functions
def flatten(alist):
return list(itertools.chain(*alist))
def sign(number):
"""Returns the sign of a given number."""
if number != 0:
return number / abs(number)
return 0
# Parser functions
def tokenize_string(astr):
"""Break a string down into a list of tokens. A token is a set, a range or
a set of characters that gives a unique string. Examples:
"{1,2,3}.com" ~> ["", "{1,2,3}", ".com"]
"easy as {a,b,c} or [1-3]" ~> ["easy as ", "{a,b,c}", " or ", "[1-3]"]
tokens = [astr]
tokens = flatten(map(lambda s: re.split(regex, s), tokens))
return tokens
def parse_token(token):
"""Given a single 'token' element, return a list of values that the token
expands to. For example:
{1,2,3} ~> [1, 2, 3]
[1-10] ~> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
[17-5:-3] ~> [17, 14, 11, 8, 5]
[a-z:5] ~> [a, f, k, p, u, z]
abc ~> [abc]
All entries in the returned list are strings. Single strings are returned
as a singleton.
if ITEMISED_SET.match(token):
return token[1:-1].split(',')
elif LOWERCASE_RANGE.match(token) or UPPERCASE_RANGE.match(token) or \
# Get the step from the range, or default to 1.
if ':' in token:
step = int(token[1:-1].split(':')[-1])
step = 1
token = token[1:-1].split(':')[0]
start, end = token.split('-')
# Capture the length of the string so we can include leading zeros.
l_zeros = len(start)
# Python ranges are half-open, whereas the given ranges are always
# closed at both ends. We take the sign of the step to determine how to
# adjust the range, because the range may be descending.
if start.isdigit():
start = int(start)
end = int(end) + sign(step)
return map(lambda s: str(s).zfill(l_zeros), range(start, end, step))
return map(chr, range(ord(start), ord(end) + sign(step), step))
# If it doesn't match one of the previous expressions, then the token is a
# plain string.
return [token]
def parse_string(astr):
"""Take a string formatted in cURL syntax and return a list of strings
which it expands to.
token_lists = map(parse_token, tokenize_string(astr))
return itertools.imap(lambda p: ''.join(p), itertools.product(*token_lists))
def main():
"""Print a list of strings that arise from a single cURL format string."""
import sys
print '\n'.join(list(parse_string(sys.argv[1])))
if __name__ == '__main__':
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment