iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-24 01:41:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix documentation in wordfreq_builder.tokenizers

Browse Source 
Former-commit-id: 8ddc19a5ca
This commit is contained in:
Rob Speer 2016-01-13 15:18:12 -05:00
parent 56f830d678
commit ee8cfb5a50
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
wordfreq_builder/wordfreq_builder/tokenizers.py
View File

@ -54,11 +54,17 @@ KEEP_THESE_LANGUAGES = {
def cld2_reddit_tokenizer(text):
"""
A language-detecting tokenizer with special cases for handling text from
Reddit.
"""
text = URL_RE.sub('', text)
text = MARKDOWN_URL_RESIDUE_RE.sub(']', text)
lang = cld2_detect_language(text)
if lang not in KEEP_THESE_LANGUAGES:
# Reddit is 99.9% English, so if we detected a rare language, it's
# much more likely that it's actually English.
lang = 'en'
tokens = tokenize(text, lang, include_punctuation=True)
@ -86,7 +92,7 @@ def tokenize_by_language(in_filename, out_prefix, tokenizer):
"""
Process a file by running it through a given tokenizer.
Produces output files that are separated by language, with newlines
Produces output files that are separated by language, with spaces
between the tokens.
"""
out_files = {}