gistfile1.clj

(ns net.licenser.jsinterpreter.jsparser
  (:gen-class))

(defstruct token-struct :kind :value)
(def token (partial struct token-struct))

(defn re-scan
  [regexp string]
  (let [regexp (re-pattern (str "(?s)^(" regexp ")(.*)"))]
   (next (re-find regexp string))))

(defn scan-token
  ([token-type regexp string post-processor]
    (let [match (re-scan regexp string)]
    (if match
    [(post-processor (token token-type (first match))) (second match)]
      nil)))
  ([token-type regexp string]
    (scan-token token-type regexp string (fn [x] x))))

(def keywords #{"break" "continue" "do" "for" "import" "new" "this" "void"
                "case" "default" "else" "function" "in" "return" "typeof"
                "while" "comment" "delete" "export" "if" "label" "switch" "var"
                "with" "abstract" "implements" "protected" "boolean"
                "instanceOf" "public" "byte" "int" "short" "char" "interface"
                "static" "double" "long" "synchronized" "false" "native"
                "throws" "final" "null" "transient" "float" "package" "true"
                "goto" "private" "catch" "enum" "throw" "class" "extends"
                "try" "const" "finally" "debugger" "super"})

(defn tokenize
  [s]
  (let [match (or
                (scan-token :comment #"(?m-s://.*$)" s)
                (scan-token :comment #"/\*.*\*/" s)
                (scan-token :newline #"\n" s)
                (scan-token :string #"'[^\\']*(?:\\(?:u[a-zA-Z0-9]{4}|[a-tv-wyz1-9_\W])[^\\']*)*'" s)
                (scan-token :string #"\"[^\\\"]*(?:\\(?: u[a-zA-Z0-9]{4}|[a-tv-wyz1-9_\W])[^\\\"]*)*\"" s)
                (scan-token :float #"(?:\d*\.\d+)(?:e[-+]?\d+)?|\d+e[-+]?\d+" s)
                (scan-token :space #"\s+" s)
                (scan-token :integer #"0x[0-9A-F]+|\d+"  s)
                (scan-token :par #"[\[\]\{\}]" s)
                (scan-token :ident #"\$?[a-z_][a-z_0-9]*" s (fn [t] 
                                                              (if 
                                                                (keywords (:value t))
                                                                (token :keyword (:value t))
                                                                t))))]
    (if match
      (conj (tokenize (second match)) (first match))
      '())))


(tokenize "false")