53 lines
1.7 KiB
Python
53 lines
1.7 KiB
Python
"""
|
|
The implementations are adapted from https://github.com/tylin/coco-caption/
|
|
blob/master/pycocoevalcap/meteor/meteor.py
|
|
"""
|
|
|
|
import os
|
|
import subprocess
|
|
import tempfile
|
|
|
|
METEOR_JAR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|
'meteor-1.5.jar')
|
|
|
|
|
|
class Meteor(object):
|
|
"""
|
|
The implementation of Meteor refer to 'METEOR: An automatic metric for MT
|
|
evaluation with improved correlation with human judgments'
|
|
[Banerjee S, Lavie A., 2005]
|
|
(https://aclanthology.org/W05-0909.pdf)
|
|
"""
|
|
def __init__(self):
|
|
self.meteor_cmd = ' '.join([
|
|
'java', '-Xmx2G', '-jar', METEOR_JAR, '{pred}', '{reference}',
|
|
'-l', 'en', '-norm'
|
|
])
|
|
|
|
def compute_score(self, gts, res):
|
|
assert (gts.keys() == res.keys())
|
|
imgIds = gts.keys()
|
|
# Clean up a NamedTemporaryFile on your own
|
|
# delete=True means the file will be deleted on close
|
|
pred_tmp = tempfile.NamedTemporaryFile(mode='w', dir='./', delete=True)
|
|
ref_tmp = tempfile.NamedTemporaryFile(mode='w', dir='./', delete=True)
|
|
for i in imgIds:
|
|
assert (len(res[i]) == 1) # only one prediction per example
|
|
# do stuff with temp
|
|
pred_tmp.write('{}\n'.format(res[i][0]))
|
|
ref_tmp.write('{}\n'.format(gts[i][0]))
|
|
|
|
pred_tmp.flush()
|
|
ref_tmp.flush()
|
|
|
|
output = subprocess.getoutput(
|
|
self.meteor_cmd.format(pred=pred_tmp.name, reference=ref_tmp.name))
|
|
score = float(output.split('\n')[-1].split(':')[-1].strip())
|
|
pred_tmp.close() # deletes the file
|
|
ref_tmp.close() # deletes the file
|
|
|
|
return score, None
|
|
|
|
def method(self):
|
|
return "METEOR"
|