Add nltk.download('punkt_tab') to Rouge metric.

This is needed to use the nltk.sent_tokenize() downstream. PiperOrigin-RevId: 686987036
tensorflow · Oct 17, 2024 · 7a068ec · 7a068ec
1 parent 181ac2f
commit 7a068ec
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 0 deletions.
diff --git a/RELEASE.md b/RELEASE.md
@@ -19,6 +19,7 @@
 *   Modifies a ROUGE Test to be compatible with NumPy v2.0.1.
 *   Remove keras_util_test.py which is based on estimator models.
 *   Remove dependency on eval_saved_model encodings.
+*   Downloads `punkt_tab` in Rouge metric.
 
 ## Breaking Changes
 

diff --git a/tensorflow_model_analysis/metrics/rouge.py b/tensorflow_model_analysis/metrics/rouge.py
@@ -93,6 +93,7 @@ def setup(self):
       if not tokenizer_installed:
         logging.info(_LOGGING_MESSAGE_TOKENIZER_PREPARER)
         nltk.download('punkt')
+        nltk.download('punkt_tab')
 
   def create_accumulator(self) -> _Accumulator:
     return _Accumulator()