Don't support opcodes for levenshtein (missing or incorrectly implemented in all tested libraries)

Author: MikeSmithEU

src/benchmarkstt/diff/core.py

@@ -54,12 +54,15 @@ def __init__(self, a, b, **kwargs):
         self._matcher = edit_distance.SequenceMatcher(**self._kwargs)
 
     def get_opcodes(self):
-        return self.simplify_opcodes(self._matcher.get_opcodes())
+        raise NotImplementedError("not supported by %r" % (self,))
 
     def get_error_rate(self):
         a = self._kwargs['a']
         b = self._kwargs['b']
-        return editdistance.eval(a, b) / len(a)
+        len_a = len(a)
+        if len_a == 0:
+            return 0 if len(b) == 0 else 1
+        return editdistance.eval(a, b) / len_a
 
     @staticmethod
     def simplify_opcodes(opcodes):

src/benchmarkstt/metrics/core.py

@@ -100,16 +100,19 @@ def __init__(self, mode=None, differ_class: Union[str, type_differ, None] = None
     def compare(self, ref: type_schema, hyp: type_schema) -> float:
         diffs = get_differ(ref, hyp, differ_class=self._differ_class)
 
-        counts = diffs.get_opcode_counts()
-
-        changes = counts.replace * self.SUB_PENALTY + \
-            counts.delete * self.DEL_PENALTY + \
-            counts.insert * self.INS_PENALTY
-
-        total = counts.equal + counts.replace + counts.delete
-        if total == 0:
-            return 1 if changes else 0
-        return changes / total
+        try:
+            counts = diffs.get_opcode_counts()
+
+            changes = counts.replace * self.SUB_PENALTY + \
+                counts.delete * self.DEL_PENALTY + \
+                counts.insert * self.INS_PENALTY
+
+            total = counts.equal + counts.replace + counts.delete
+            if total == 0:
+                return 1 if changes else 0
+            return changes / total
+        except NotImplementedError:
+            return diffs.get_error_rate()
 
 
 class CER(Metric):

tests/benchmarkstt/test_diff.py

@@ -22,11 +22,11 @@ def clean_opcodes(opcodes):
     return list(map(clean_opcode, opcodes))
 
 
-def test_simple_levenshtein_ratcliff_similarity():
-    a = list('012345')
-    b = list('023x45')
-    assert(clean_opcodes(Levenshtein(a, b).get_opcodes()) ==
-           clean_opcodes(RatcliffObershelp(a, b).get_opcodes()))
+# def test_simple_levenshtein_ratcliff_similarity():
+#     a = list('012345')
+#     b = list('023x45')
+#     assert(clean_opcodes(Levenshtein(a, b).get_opcodes()) ==
+#            clean_opcodes(RatcliffObershelp(a, b).get_opcodes()))
 
 
 @differs_decorator