prateekjoshi565/testing_tokenizer_bert.py

## testing_tokenizer_bert.py
# sample data
text = ["this is a bert model tutorial", "we will fine-tune a bert model"]

# encode text
sent_id = tokenizer.batch_encode_plus(text, padding=True)

# output
print(sent_id)
	# sample data
	text = ["this is a bert model tutorial", "we will fine-tune a bert model"]

	# encode text
	sent_id = tokenizer.batch_encode_plus(text, padding=True)

	# output
	print(sent_id)