UCDP-AEC

Models and data for the UCDP Abstractive Event analysis Corpus paper
git clone https://esimon.eu/repos/UCDP-AEC.git
Log | Files | Refs | README | LICENSE

sentence_splitter.py (1412B)

      1 # Copyright 2020 The HuggingFace Team. All rights reserved.
      2 #
      3 # Licensed under the Apache License, Version 2.0 (the "License");
      4 # you may not use this file except in compliance with the License.
      5 # You may obtain a copy of the License at
      6 #
      7 #     http://www.apache.org/licenses/LICENSE-2.0
      8 #
      9 # Unless required by applicable law or agreed to in writing, software
     10 # distributed under the License is distributed on an "AS IS" BASIS,
     11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     12 # See the License for the specific language governing permissions and
     13 # limitations under the License.
     14 import os
     15 import re
     16 
     17 from filelock import FileLock
     18 
     19 try:
     20     import nltk
     21 
     22     NLTK_AVAILABLE = True
     23 except (ImportError, ModuleNotFoundError):
     24     NLTK_AVAILABLE = False
     25 
     26 # if NLTK_AVAILABLE:
     27 #     if not os.path.exists("~/nltk_data/tokenizers/punkt"):
     28 #         print('Start download punk')
     29 #         with FileLock(".lock") as lock:
     30 #             nltk.download("punkt", quiet=True)
     31 #     else:
     32 #         print('Skip download punk')
     33 
     34 
     35 def add_newline_to_end_of_each_sentence(x: str) -> str:
     36     """This was added to get rougeLsum scores matching published rougeL scores for BART and PEGASUS."""
     37     re.sub("<n>", "", x)  # remove pegasus newline char
     38     assert NLTK_AVAILABLE, "nltk must be installed to separate newlines between sentences. (pip install nltk)"
     39     return "\n".join(nltk.sent_tokenize(x))