limit Reddit data to just English

Former-commit-id: 2276d97368
2024-12-23 09:21:37 +00:00 · 2016-04-15 17:01:21 -04:00 · 2016-04-15 17:01:21 -04:00 · a0d93e0ce8
commit a0d93e0ce8
parent 5a37cc22c7
12 changed files with 4 additions and 4 deletions
--- a/wordfreq/data/combined_de.msgpack.gz
+++ b/wordfreq/data/combined_de.msgpack.gz
--- a/wordfreq/data/combined_en.msgpack.gz
+++ b/wordfreq/data/combined_en.msgpack.gz
--- a/wordfreq/data/combined_es.msgpack.gz
+++ b/wordfreq/data/combined_es.msgpack.gz
--- a/wordfreq/data/combined_sv.msgpack.gz
+++ b/wordfreq/data/combined_sv.msgpack.gz
--- a/wordfreq/data/large_de.msgpack.gz
+++ b/wordfreq/data/large_de.msgpack.gz
--- a/wordfreq/data/large_en.msgpack.gz
+++ b/wordfreq/data/large_en.msgpack.gz
--- a/wordfreq/data/large_es.msgpack.gz
+++ b/wordfreq/data/large_es.msgpack.gz
--- a/wordfreq/data/twitter_de.msgpack.gz
+++ b/wordfreq/data/twitter_de.msgpack.gz
--- a/wordfreq/data/twitter_en.msgpack.gz
+++ b/wordfreq/data/twitter_en.msgpack.gz
--- a/wordfreq/data/twitter_es.msgpack.gz
+++ b/wordfreq/data/twitter_es.msgpack.gz
--- a/wordfreq/data/twitter_sv.msgpack.gz
+++ b/wordfreq/data/twitter_sv.msgpack.gz
--- a/wordfreq_builder/wordfreq_builder/config.py
+++ b/wordfreq_builder/wordfreq_builder/config.py
@ -42,10 +42,10 @@ CONFIG = {
        'subtlex-other': ['de', 'nl', 'zh'],
        'jieba': ['zh'],

-        # About 99.2% of Reddit is in English, but there are pockets of
-        # conversation in other languages. These are the languages that seem
-        # to have enough non-spam comments to actually learn from.
-        'reddit': ['de', 'en', 'es', 'sv']
+        # About 99.2% of Reddit is in English. There are pockets of
+        # conversation in other languages, but we're concerned that they're not
+        # representative enough for learning general word frequencies.
+        'reddit': ['en']
    },
    # Subtlex languages that need to be pre-processed
    'wordlist_paths': {