tokenization with expert.ai NL API

token_subdivison.py

text = "CNBC has commented on the robot's  lifelike skin and her ability to emulate more than 60 facial expressions." 
language= 'en'

output = client.specific_resource_analysis(
    body={"document": {"text": text}}, 
    params={'language': language, 'resource': 'disambiguation'
})

#to print to tokens within the sentence
print (f'{"TOKEN":{20}} ')
print (f'{"----":{20}}') 

#we use the element output.tokens to subdivise the text into tokens
#start and end are the positions of the token in the analyzed text
for token in output.tokens:
    print (f'{text[token.start:token.end]:{20}}')