Split a string into tokens with a given separator supporting escape sequences

tokenize.go

package main

import (
	"errors"
	"fmt"
)

func tokenizeString(s string, sep, escape rune) (tokens []string, err error) {
	var runes []rune
	inEscape := false
	for _, r := range s {
		switch {
		case inEscape:
			inEscape = false
			fallthrough
		default:
			runes = append(runes, r)
		case r == escape:
			inEscape = true
		case r == sep:
			tokens = append(tokens, string(runes))
			runes = runes[:0]
		}
	}
	tokens = append(tokens, string(runes))
	if inEscape {
		err = errors.New("invalid terminal escape")
	}
	return tokens, err
}

func main() {
	const sample = `one \ two three`
	const separator = ' '
	const escape = '\\'

	fmt.Printf("Input:   %q\n", sample)
	tokens, err := tokenizeString(sample, separator, escape)
	if err != nil {
		fmt.Println("error:", err)
	} else {
		fmt.Printf("Tokens: %q\n", tokens)
	}
}

This is most useful when say you want to split a string up into tokens (tokenize) separated by a space but allow for the separator to be escaped.

Example:

$ go run main.go
Input:   "one \\ two three"
Tokens: ["one" " two" "three"]