iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-24 09:51:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

removes arabic commas

Browse Source 
Former-commit-id: 83797bd276
This commit is contained in:
Joshua Chin 2015-06-25 12:02:59 -04:00
parent 78bff813e3
commit 60782d3796
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
wordfreq/__init__.py
View File

@ -118,6 +118,10 @@ def tokenize(text, lang):
except NameError: except NameError:
from wordfreq.mecab import mecab_tokenize from wordfreq.mecab import mecab_tokenize
return mecab_tokenize(text) return mecab_tokenize(text)
elif lang == 'ar':
tokens = simple_tokenize(text)
tokens = [token.replace('ـ', '') for token in tokens]
return [token for token in tokens if token] # remove empty strings
else: else:
return simple_tokenize(text) return simple_tokenize(text)