diff --git a/wordfreq_builder/wordfreq_builder/tokenizers.py b/wordfreq_builder/wordfreq_builder/tokenizers.py index af5d115..25674e7 100644 --- a/wordfreq_builder/wordfreq_builder/tokenizers.py +++ b/wordfreq_builder/wordfreq_builder/tokenizers.py @@ -48,8 +48,8 @@ def cld2_surface_tokenizer(text): # Low-frequency languages tend to be detected incorrectly. Keep a limited # list of languages we're allowed to use here. KEEP_THESE_LANGUAGES = { - 'ar', 'de', 'el', 'en', 'es', 'fr', 'hr', 'id', 'ja', 'ko', 'ms', 'nl', - 'pl', 'pt', 'ro', 'ru', 'sv', 'th' + 'ar', 'de', 'el', 'en', 'es', 'fr', 'hr', 'id', 'it', 'ja', 'ko', 'ms', + 'nl', 'pl', 'pt', 'ro', 'ru', 'sv', 'th' }