This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
from tqdm import tqdm | |
from difflib import SequenceMatcher | |
import re | |
import pickle | |
def matcher(string, pattern): | |
''' | |
Return the start and end index of any pattern present in the text. | |
''' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
import tensorflow as tf | |
save_path = "<Your path>/pegasus/data/testdata/test_pattern_1.tfrecords" | |
input_dict = dict( | |
inputs=[ | |
# Your text inputs to be summarized. | |
], | |
targets=[ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
save_path = "<Your path>/pegasus/data/testdata/test_pattern_1.tfrecords" | |
@registry.register("test_transformer") | |
def test_transformer(param_overrides): | |
return transformer_params( | |
{ | |
"train_pattern": save_path, | |
"dev_pattern": save_path, | |
"test_pattern": save_path, | |
"max_input_len": 1024, | |
"max_output_len": 256, |