Fix edge cases in split_sentence

BishopLiu · Jun 15, 2022 · c9317ae · c9317ae
1 parent 584ce7e
commit c9317ae
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 1 deletion.
diff --git a/hanlp/utils/rules.py b/hanlp/utils/rules.py
@@ -29,7 +29,11 @@ def split_sentence(text, best=True):
             yield chunk
             continue
         processed = replace_with_separator(chunk, SEPARATOR, [AB_SENIOR, AB_ACRONYM])
-        for sentence in RE_SENTENCE.finditer(processed):
+        sents = list(RE_SENTENCE.finditer(processed))
+        if not sents:
+            yield chunk
+            continue
+        for sentence in sents:
             sentence = replace_with_separator(sentence.group(), r" ", [UNDO_AB_SENIOR, UNDO_AB_ACRONYM])
             yield sentence
 

diff --git a/tests/test_rules.py b/tests/test_rules.py
@@ -0,0 +1,17 @@
+# -*- coding:utf-8 -*-
+# Author: hankcs
+# Date: 2022-03-22 17:17
+import unittest
+
+from hanlp.utils.rules import split_sentence
+
+
+class TestRules(unittest.TestCase):
+    def test_eos(self):
+        self.assertListEqual(list(split_sentence('叶')), ['叶'])
+        self.assertListEqual(list(split_sentence('他说：“加油。”谢谢')), ['他说：“加油。”', '谢谢'])
+        self.assertListEqual(list(split_sentence('Go to hankcs.com. Yes.')), ['Go to hankcs.com.', 'Yes.'])
+
+
+if __name__ == '__main__':
+    unittest.main()