gistfile1.go

func Tokenize(s string) []string {
	out := []string{}
	for l, r := 0, 0; r < len(s); {
		for l = r; isSep(s[l]); l++ {
		}
		for r = l; r < len(s) && !isSep(s[r]); r++ {
		}
		out = append(out, s[l:r])
	}
	return out
}