init

charmatch.py

@@ -15,7 +15,7 @@
 
 import evaluate
 import datasets
-
+from Levenshtein import distance as lev
 
 # TODO: Add BibTeX citation
 _CITATION = """\
@@ -66,13 +66,14 @@ class charmatch(evaluate.Metric):
         return evaluate.MetricInfo(
             # This is the description that will appear on the modules page.
             module_type="metric",
-            description=_DESCRIPTION,
-            citation=_CITATION,
-            inputs_description=_KWARGS_DESCRIPTION,
+            description="Charmatch",
+            citation="",
+            inputs_description="input expected output",
             # This defines the format of each prediction and reference
             features=datasets.Features({
-                'predictions': datasets.Value('int64'),
-                'references': datasets.Value('int64'),
+                'input': datasets.Value('string'),
+                'expected': datasets.Value('string'),
+                'output': datasets.Value('string')
             }),
             # Homepage of the module for documentation
             homepage="http://module.homepage",
@@ -86,10 +87,17 @@ class charmatch(evaluate.Metric):
         # TODO: Download external resources if needed
         pass
 
-    def _compute(self, predictions, references):
-        """Returns the scores"""
-        # TODO: Compute the different scores of the module
-        accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions)
+    def _compute(self, input, expected, output):
+        expected_corrections = lev(input, expected)
+        distance_to_input = lev(input, output)
+        distance_to_expected = lev(output, expected)
+
+        true_positives = min(expected_corrections, max(0, (expected_corrections + distance_to_input - distance_to_expected))) / 2
+
+        precision = true_positives / distance_to_input
+        recall = true_positives / expected_corrections
+        f_05 = (1 + 0.5**2) * (precision * recall) / (0.5**2 * precision + recall)
+
         return {
-            "accuracy": accuracy,
-        }
+            "fscore": f_05
+        }

requirements.txt

@@ -1 +1,2 @@
-git+https://github.com/huggingface/evaluate@main
+git+https://github.com/huggingface/evaluate@main
+levenshtein