gistfile1.go

package parse

import (
	"bufio"
	"errors"
	"io"
	"unicode"
)

var InvalidToken = errors.New("invalid token")

const (
	Symbol = iota
	Integer
	String
	LeftParenthesis
	RightParenthesis
)

type Token struct {
	Kind  int
	Value interface{}
}

func Lex(reader io.Reader) ([]Token, error) {
	rd := bufio.NewReader(reader)
	tokens := []Token{}
	for {
		r, _, err := rd.ReadRune()
		if err != nil {
			if err == io.EOF {
				return tokens, nil
			} else {
				return nil, err
			}
		}
		switch {
		case isSpace(r):
			break

		case r == '(':
			tokens = append(tokens, Token{LeftParenthesis, nil})

		case r == ')':
			tokens = append(tokens, Token{RightParenthesis, nil})

		case isIdentifierHead(r):
			name := string(r)
			for {
				r, _, err = rd.ReadRune()
				if isIdentifierTail(r) {
					name += string(r)
				} else if err == nil || err == io.EOF {
					break
				} else {
					return nil, err
				}
			}
			rd.UnreadRune()
			tokens = append(tokens, Token{Symbol, name})

		default:
			return nil, InvalidToken
		}
	}
	return tokens, nil
}

func isSpace(r rune) bool {
	return unicode.In(r, unicode.Zs)
}

func isIdentifierHead(r rune) bool {
	return unicode.In(r, unicode.L, unicode.Sm, unicode.Pc, unicode.Pd)
}

func isIdentifierTail(r rune) bool {
	return isIdentifierHead(r) || unicode.In(r, unicode.N)
}