iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-24 01:41:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

factored out fixing arabic

Browse Source
This commit is contained in:
Joshua Chin 2015-07-17 15:39:12 -04:00
parent 303bd88ba2
commit 4e3a5263c3
1 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
wordfreq/__init__.py
View File

@ -66,11 +66,18 @@ def tokenize(text, lang):
return mecab_tokenize(text)
if lang == 'ar':
text = COMBINING_MARK_RE.sub('', text.replace('ـ', ''))
text = standardize_arabic(text)
return simple_tokenize(text)
def standardize_arabic(text):
"""
Standardizes arabic text by removing combining marks and tatweels.
"""
return COMBINING_MARK_RE.sub('', text.replace('ـ', ''))
def read_cBpack(filename):
"""
Read a file from an idiosyncratic format that we use for storing