swift_token.swift

// Import the NaturalLanguage library
import NaturalLanguage

let text = """
Analytics Vidhya provides a community based knowledge portal for Analytics and Data Science professionals. The aim of the platform is to become a complete portal serving all knowledge and career needs of Data Science Professionals.
"""

// Initialize the tokenizer with unit of "word"
let tokenizer = NLTokenizer(unit: .word)
// Set the string to be processed
tokenizer.string = text
// Loop over all the tokens and print them
tokenizer.enumerateTokens(in: text.startIndex..<text.endIndex) { tokenRange, _ in
    print(text[tokenRange])
    return true
}