Tokenization

siddhants-MacBook-Air:railsblog sid$ rails c
Loading development environment (Rails 5.2.3)
irb(main):001:0> scanner = ActionDispatch::Journey::Scanner.new
=> #<ActionDispatch::Journey::Scanner:0x00007f84f432d658 @ss=nil>
irb(main):002:0> scanner.scan_setup("moderators/:id")
=> #<StringScanner 0/14 @ "moder...">
irb(main):003:0> scanner.next_token
=> [:LITERAL, "moderators"]
irb(main):004:0> scanner.next_token
=> [:SLASH, "/"]
irb(main):005:0> scanner.next_token
=> [:SYMBOL, ":id"]
irb(main):006:0> scanner.next_token
=> nil
irb(main):007:0> scanner.next_token
=> nil
irb(main):008:0>