iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-23 17:31:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

no Thai because we can't tokenize it

Browse Source
This commit is contained in:
Rob Speer 2015-12-02 12:38:03 -05:00
parent 8f6cd0e57b
commit 95f53e295b
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
wordfreq_builder/wordfreq_builder/tokenizers.py
View File

@ -49,7 +49,7 @@ def cld2_surface_tokenizer(text):
# list of languages we're allowed to use here. # list of languages we're allowed to use here.
KEEP_THESE_LANGUAGES = { KEEP_THESE_LANGUAGES = {
'ar', 'de', 'el', 'en', 'es', 'fr', 'hr', 'id', 'it', 'ja', 'ko', 'ms', 'ar', 'de', 'el', 'en', 'es', 'fr', 'hr', 'id', 'it', 'ja', 'ko', 'ms',
'nl', 'pl', 'pt', 'ro', 'ru', 'sv', 'th' 'nl', 'pl', 'pt', 'ro', 'ru', 'sv'
} }