iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-23 17:31:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

add Polish and Swedish, which have sufficient data

Browse Source
This commit is contained in:
Rob Speer 2015-09-04 16:32:37 -04:00
parent 25edaad962
commit 447d7e5134
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
wordfreq_builder/wordfreq_builder/config.py
View File

@ -12,15 +12,15 @@ CONFIG = {
# Consider adding:
# 'th' when we get tokenization for it
# 'hi' when we stop messing up its tokenization
# 'tl' because it's probably ready right now
# 'pl' because we have 3 sources for it
# 'tl' with one more data source
# 'sv' because its data sources are ready
'twitter': [
'ar', 'de', 'el', 'en', 'es', 'fr', 'id', 'it', 'ja', 'ko', 'ms', 'nl',
'pt', 'ru', 'tr'
'pl', 'pt', 'ru', 'sv', 'tr'
],
'wikipedia': [
'ar', 'de', 'en', 'el', 'es', 'fr', 'id', 'it', 'ja', 'ko', 'ms', 'nl',
'pt', 'ru', 'tr'
'pl', 'pt', 'ru', 'sv', 'tr'
],
'opensubtitles': [
# This list includes languages where the most common word in