tokenize.py

import re

def tokenize(sentence):
    delimiter = "(plus|minus|divided by|times|\(|\))"
    tokens = re.split(delimiter, sentence)
    # Buang spasi di awal dan akhir
    tokens = [token.strip() for token in tokens]
    # Buang yg kosong
    tokens = list(filter(None, tokens))
    return tokens

if __name__ == "__main__":
    test = ["one plus two",
            "minus (forty-five minus five) divided by five",
            "six hundred forty-five minus nine hundred"]
    for sentence in test:
        tokens = tokenize(sentence)
        print(tokens)

kurang kerjaan banget gw ya