removes arabic commas

Former-commit-id: 83797bd276
2024-12-23 17:31:41 +00:00 · 2015-06-25 12:02:59 -04:00 · 2015-06-25 12:02:59 -04:00 · 60782d3796
commit 60782d3796
parent 78bff813e3
1 changed files with 4 additions and 0 deletions
--- a/wordfreq/init.py
+++ b/wordfreq/init.py
@ -118,6 +118,10 @@ def tokenize(text, lang):
        except NameError:
            from wordfreq.mecab import mecab_tokenize
            return mecab_tokenize(text)
+    elif lang == 'ar':
+        tokens = simple_tokenize(text)
+        tokens = [token.replace('ـ', '') for token in tokens]
+        return [token for token in tokens if token] # remove empty strings
    else:
        return simple_tokenize(text)