string lexer

lexer.scm

(use-modules (ice-9 rdelim) (rnrs))

(define (k f init val pred)
(if (pred val)
    init
    (call-with-values (lambda () (f init val))
      (lambda (lst rst) (k f lst rst pred)))))

(define (lexer str delimiters)
 (define (-> c)
  (if (eof-object? c) '() (list (string c))))
 (define (tokenizer lst str)
  (call-with-input-string str
   (lambda (port)
    (let* ((token (read-delimited delimiters port 'peek))
           (delim (-> (read-char port)))  
           (rest (get-string-all port)))
         (values (if (string-null? token) `(,@lst ,@delim) `(,@lst ,token ,@delim))
                 rest)))))
  (k tokenizer '() str string-null?))
  

(lexer "hello.x+(bye)"  ".+()")

==> ("hello" "." "x" "+" "(" "bye" ")")