iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-23 17:31:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

removes arabic commas

Browse Source
This commit is contained in:
Joshua Chin 2015-06-25 12:02:59 -04:00
parent 6e1f7e30c6
commit 83797bd276
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
wordfreq/__init__.py
View File

@ -118,6 +118,10 @@ def tokenize(text, lang):
except NameError:
from wordfreq.mecab import mecab_tokenize
return mecab_tokenize(text)
elif lang == 'ar':
tokens = simple_tokenize(text)
tokens = [token.replace('ـ', '') for token in tokens]
return [token for token in tokens if token] # remove empty strings
else:
return simple_tokenize(text)