Fix calculation error when ref is empty

.gitignore

@@ -21,4 +21,4 @@ multi-bleu.perl
 glove2word2vec.py
 
 .idea/
-venv/
+venv/

nlgeval/__init__.py

@@ -225,7 +225,6 @@ def load_scorers(self):
         if 'CIDEr' not in self.metrics_to_omit:
             self.scorers.append((Cider(), "CIDEr"))
 
-
     def load_skipthought_model(self):
         from nlgeval.skipthoughts import skipthoughts
         import numpy as np
@@ -254,6 +253,7 @@ def compute_individual_metrics(self, ref, hyp):
         hyp_list = [hyp]
 
         ret_scores = {}
+
         if not self.no_overlap:
             for scorer, method in self.scorers:
                 score, scores = scorer.compute_score(refs, hyps)
@@ -266,8 +266,10 @@ def compute_individual_metrics(self, ref, hyp):
         if not self.no_skipthoughts:
             vector_hyps = self.skipthought_encoder.encode([h.strip() for h in hyp_list], verbose=False)
             ref_list_T = self.np.array(ref_list).T.tolist()
-            vector_refs = map(lambda refl: self.skipthought_encoder.encode([r.strip() for r in refl], verbose=False), ref_list_T)
-            cosine_similarity = list(map(lambda refv: self.cosine_similarity(refv, vector_hyps).diagonal(), vector_refs))
+            vector_refs = map(lambda refl: self.skipthought_encoder.encode([r.strip() for r in refl], verbose=False),
+                              ref_list_T)
+            cosine_similarity = list(
+                map(lambda refv: self.cosine_similarity(refv, vector_hyps).diagonal(), vector_refs))
             cosine_similarity = self.np.max(cosine_similarity, axis=0).mean()
             ret_scores['SkipThoughtCS'] = cosine_similarity
 
@@ -304,8 +306,10 @@ def compute_metrics(self, ref_list, hyp_list):
         if not self.no_skipthoughts:
             vector_hyps = self.skipthought_encoder.encode([h.strip() for h in hyp_list], verbose=False)
             ref_list_T = self.np.array(ref_list).T.tolist()
-            vector_refs = map(lambda refl: self.skipthought_encoder.encode([r.strip() for r in refl], verbose=False), ref_list_T)
-            cosine_similarity = list(map(lambda refv: self.cosine_similarity(refv, vector_hyps).diagonal(), vector_refs))
+            vector_refs = map(lambda refl: self.skipthought_encoder.encode([r.strip() for r in refl], verbose=False),
+                              ref_list_T)
+            cosine_similarity = list(
+                map(lambda refv: self.cosine_similarity(refv, vector_hyps).diagonal(), vector_refs))
             cosine_similarity = self.np.max(cosine_similarity, axis=0).mean()
             ret_scores['SkipThoughtCS'] = cosine_similarity
 

nlgeval/pycocoevalcap/cider/cider_scorer.py

@@ -10,6 +10,7 @@
 from six.moves import xrange as range
 import six
 
+
 def precook(s, n=4, out=False):
     """
     Takes a string as input and returns an object that can be given to
@@ -21,13 +22,14 @@ def precook(s, n=4, out=False):
     """
     words = s.split()
     counts = defaultdict(int)
-    for k in range(1,n+1):
-        for i in range(len(words)-k+1):
-            ngram = tuple(words[i:i+k])
+    for k in range(1, n + 1):
+        for i in range(len(words) - k + 1):
+            ngram = tuple(words[i:i + k])
             counts[ngram] += 1
     return counts
 
-def cook_refs(refs, n=4): ## lhuang: oracle will call with "average"
+
+def cook_refs(refs, n=4):  ## lhuang: oracle will call with "average"
     '''Takes a list of reference sentences for a single segment
     and returns an object that encapsulates everything that BLEU
     needs to know about them.
@@ -37,6 +39,7 @@ def cook_refs(refs, n=4): ## lhuang: oracle will call with "average"
     '''
     return [precook(ref, n) for ref in refs]
 
+
 def cook_test(test, n=4):
     '''Takes a test sentence and returns an object that
     encapsulates everything that BLEU needs to know about it.
@@ -46,6 +49,7 @@ def cook_test(test, n=4):
     '''
     return precook(test, n, True)
 
+
 class CiderScorer(object):
     """CIDEr scorer.
     """
@@ -73,9 +77,9 @@ def cook_append(self, test, refs):
         if refs is not None:
             self.crefs.append(cook_refs(refs))
             if test is not None:
-                self.ctest.append(cook_test(test)) ## N.B.: -1
+                self.ctest.append(cook_test(test))  ## N.B.: -1
             else:
-                self.ctest.append(None) # lens of crefs and ctest have to match
+                self.ctest.append(None)  # lens of crefs and ctest have to match
 
     def size(self):
         assert len(self.crefs) == len(self.ctest), "refs/test mismatch! %d<>%d" % (len(self.crefs), len(self.ctest))
@@ -92,6 +96,7 @@ def __iadd__(self, other):
             self.crefs.extend(other.crefs)
 
         return self
+
     def compute_doc_freq(self):
         '''
         Compute term frequency for reference data.
@@ -101,7 +106,7 @@ def compute_doc_freq(self):
         '''
         for refs in self.crefs:
             # refs, k ref captions of one image
-            for ngram in set([ngram for ref in refs for (ngram,count) in six.iteritems(ref)]):
+            for ngram in set([ngram for ref in refs for (ngram, count) in six.iteritems(ref)]):
                 self.document_frequency[ngram] += 1
             # maxcounts[ngram] = max(maxcounts.get(ngram,0), count)
 
@@ -117,13 +122,13 @@ def counts2vec(cnts):
             vec = [defaultdict(float) for _ in range(self.n)]
             length = 0
             norm = [0.0 for _ in range(self.n)]
-            for (ngram,term_freq) in six.iteritems(cnts):
+            for (ngram, term_freq) in six.iteritems(cnts):
                 # give word count 1 if it doesn't appear in reference corpus
                 df = np.log(max(1.0, self.document_frequency[ngram]))
                 # ngram index
-                n = len(ngram)-1
+                n = len(ngram) - 1
                 # tf (term_freq) * idf (precomputed idf) for n-grams
-                vec[n][ngram] = float(term_freq)*(self.ref_len - df)
+                vec[n][ngram] = float(term_freq) * (self.ref_len - df)
                 # compute norm for the vector.  the norm will be used for computing similarity
                 norm[n] += pow(vec[n][ngram], 2)
 
@@ -148,16 +153,16 @@ def sim(vec_hyp, vec_ref, norm_hyp, norm_ref, length_hyp, length_ref):
             val = np.array([0.0 for _ in range(self.n)])
             for n in range(self.n):
                 # ngram
-                for (ngram,count) in six.iteritems(vec_hyp[n]):
+                for (ngram, count) in six.iteritems(vec_hyp[n]):
                     # vrama91 : added clipping
                     val[n] += min(vec_hyp[n][ngram], vec_ref[n][ngram]) * vec_ref[n][ngram]
 
                 if (norm_hyp[n] != 0) and (norm_ref[n] != 0):
-                    val[n] /= (norm_hyp[n]*norm_ref[n])
+                    val[n] /= (norm_hyp[n] * norm_ref[n])
 
-                assert(not math.isnan(val[n]))
+                assert (not math.isnan(val[n]))
                 # vrama91: added a length based gaussian penalty
-                val[n] *= np.e**(-(delta**2)/(2*self.sigma**2))
+                val[n] *= np.e ** (-(delta ** 2) / (2 * self.sigma ** 2))
             return val
 
         # compute log reference length
@@ -186,9 +191,9 @@ def compute_score(self, option=None, verbose=0):
         # compute idf
         self.compute_doc_freq()
         # assert to check document frequency
-        assert(len(self.ctest) >= max(self.document_frequency.values()))
+        assert (len(self.ctest) >= max(self.document_frequency.values()))
         # compute cider score
         score = self.compute_cider()
         # debug
         # print score
-        return np.mean(np.array(score)), np.array(score)
+        return np.mean(np.array(score)), np.array(score)