Skip to content

Instantly share code, notes, and snippets.

@allevato
Last active May 25, 2020
Embed
What would you like to do?
Dumping tokens with SwiftSyntax
import Foundation
import SwiftSyntax
let sourcePath =
CommandLine.arguments.count > 1 ? CommandLine.arguments[1] : #file
let sourceURL = URL(fileURLWithPath: sourcePath)
let sourceFile = try! SyntaxParser.parse(sourceURL)
let converter = SourceLocationConverter(
file: sourceURL.lastPathComponent, tree: sourceFile)
/// Token kinds that represent strings with different possible values store
/// those values in the enum. Since we print the token text for every token,
/// chop off the associated value part of the string if it exists.
func tokenKind(of token: TokenSyntax) -> String {
let kind = String(describing: token.tokenKind)
if let leftParen = kind.firstIndex(of: "(") {
return String(kind[..<leftParen])
}
return kind
}
/// Returns true if the trivia piece contains carriage returns or line feeds.
func isLineFeedLike(_ triviaPiece: TriviaPiece) -> Bool {
switch triviaPiece {
case .carriageReturnLineFeeds, .carriageReturns, .newlines: return true
default: return false
}
}
/// Returns true if the trivia piece is a set of spaces or tabs.
func isSpaceLike(_ triviaPiece: TriviaPiece) -> Bool {
switch triviaPiece {
case .spaces, .tabs: return true
default: return false
}
}
final class TokenVisitor: SyntaxVisitor {
override func visit(_ token: TokenSyntax) -> SyntaxVisitorContinueKind {
let isStartOfLine =
token.leadingTrivia.contains(where: isLineFeedLike)
|| token.previousToken == nil
let hasLeadingSpace =
token.leadingTrivia.contains(where: isSpaceLike)
|| token.previousToken?.trailingTrivia.contains(where: isSpaceLike) == true
// By default, this returns the location *after* any leading trivia; that
// is, the location of the first non-whitespace non-comment character in the
// token.
let loc = token.startLocation(converter: converter)
print("\(tokenKind(of: token))", terminator: "")
print("\t'\(token.text)'", terminator: "")
if isStartOfLine {
print("\t[StartOfLine]", terminator: "")
}
if hasLeadingSpace {
print("\t[LeadingSpace]", terminator: "")
}
print("\tLoc=<\(loc.file ?? ""):\(loc.line ?? 0):\(loc.column ?? 0)>")
// Tokens never have children anyway.
return .skipChildren
}
}
TokenVisitor().walk(sourceFile)
// Example output:
//
// importKeyword 'import' Loc=<main.swift:1:1>
// identifier 'Foundation' [LeadingSpace] Loc=<main.swift:1:8>
// importKeyword 'import' [StartOfLine] Loc=<main.swift:2:1>
// identifier 'SwiftSyntax' [LeadingSpace] Loc=<main.swift:2:8>
// letKeyword 'let' [StartOfLine] Loc=<main.swift:4:1>
// identifier 'sourcePath' [LeadingSpace] Loc=<main.swift:4:5>
// equal '=' [LeadingSpace] Loc=<main.swift:4:16>
// identifier 'CommandLine' [StartOfLine] [LeadingSpace] Loc=<main.swift:5:3>
// period '.' Loc=<main.swift:5:14>
// identifier 'arguments' Loc=<main.swift:5:15>
// period '.' Loc=<main.swift:5:24>
// identifier 'count' Loc=<main.swift:5:25>
// spacedBinaryOperator '>' [LeadingSpace] Loc=<main.swift:5:31>
// integerLiteral '1' [LeadingSpace] Loc=<main.swift:5:33>
// infixQuestionMark '?' [LeadingSpace] Loc=<main.swift:5:35>
// identifier 'CommandLine' [LeadingSpace] Loc=<main.swift:5:37>
// period '.' Loc=<main.swift:5:48>
// identifier 'arguments' Loc=<main.swift:5:49>
// leftSquareBracket '[' Loc=<main.swift:5:58>
// integerLiteral '1' Loc=<main.swift:5:59>
// rightSquareBracket ']' Loc=<main.swift:5:60>
// colon ':' [LeadingSpace] Loc=<main.swift:5:62>
// poundFileKeyword '#file' [LeadingSpace] Loc=<main.swift:5:64>
// ...
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment