refactor basic tokenizer

Author: sagorbrur

bnlp/tokenizer/basic.py

@@ -65,7 +65,6 @@ class BasicTokenizer:
     def tokenize(self, text):
         """Tokenizes a piece of text."""
         text = convert_to_unicode(text)
-        # handle (.) in bangla text
 
         orig_tokens = whitespace_tokenize(text)
         # print("original tokens: ", orig_tokens)
@@ -75,7 +74,6 @@ def tokenize(self, text):
 
         # print("split tokens: ", split_tokens)
         output_tokens = whitespace_tokenize(" ".join(split_tokens))
-        # get (.) back in output tokens
         return output_tokens
 
     def _run_strip_accents(self, text):