fitbert-syntax

fitbert-syntax1.py

# example from "Targeted Syntactic Evaluation of Language Models"
# https://arxiv.org/abs/1808.09031

masked_string = "the author that the guard likes ***mask***"
options = ['laugh', 'laughs']
fb.rank_with_prob(masked_string, options)

# >>> (['laughs', 'laugh'], [4.14195717654553e-12, 3.3748110100755013e-13])

fitbert-syntax2.py

# example from "Assessing the Ability of LSTMs to Learn Syntax-Sensitive Dependencies"
# https://transacl.org/ojs/index.php/tacl/article/view/972

masked_string = "accusations of abusive sockpuppetry from a trusted source ***mask*** a serious chilling effect ."
options = ["have", "has"]
fb.rank_with_prob(masked_string, options)

# >>> (['have', 'has'], [0.8899502754211426, 0.004103286191821098])