show_tokenized_result.py (1585B)
1 from transformers import AutoTokenizer 2 from extraction.event_schema import EventSchema 3 import json 4 import argparse 5 6 7 def main(): 8 parser = argparse.ArgumentParser() 9 parser.add_argument('-m', '--model', default='t5-base') 10 parser.add_argument('-d', '--data', 11 default='data/text2tree/dyiepp_ace2005_subtype') 12 options = parser.parse_args() 13 14 tokenizer = AutoTokenizer.from_pretrained(options.model, use_fast=False) 15 16 tokenizer.add_special_tokens( 17 {"additional_special_tokens": ["<extra_id_0>", "<extra_id_1>"]} 18 ) 19 20 print(tokenizer) 21 print(type(tokenizer)) 22 print("================================") 23 24 folder_path = options.data 25 26 schema_file = folder_path + "/event.schema" 27 28 event_schema = EventSchema.read_from_file(schema_file) 29 30 subtoken_list = list() 31 for typename in event_schema.type_list: 32 typename = typename.replace('_', ' ') 33 after_tokenzied = tokenizer.encode(typename, add_special_tokens=False) 34 subtoken_list += after_tokenzied 35 print(typename, after_tokenzied, 36 tokenizer.convert_ids_to_tokens(after_tokenzied)) 37 print("----------------") 38 39 print("================================") 40 41 for instance in open(folder_path + "/val.json").readlines()[:10]: 42 instance = json.loads(instance) 43 print(tokenizer.tokenize(instance['event'])) 44 print("----------------") 45 46 for name in ['<extra_id_0>', '<extra_id_1>']: 47 print(name, tokenizer.encode(name), tokenizer.tokenize(name)) 48 49 50 if __name__ == "__main__": 51 main()