iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-23 17:31:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Separate tokens with spaces, not line breaks, in intermediate files

Browse Source
This commit is contained in:
Rob Speer 2016-01-12 12:59:18 -05:00
parent 973caca253
commit 115c74583e
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
wordfreq_builder/wordfreq_builder/tokenizers.py
View File

@ -95,7 +95,7 @@ def tokenize_by_language(in_filename, out_prefix, tokenizer):
text = line.split('\t')[-1].strip() text = line.split('\t')[-1].strip()
language, tokens = tokenizer(text) language, tokens = tokenizer(text)
if language != 'un': if language != 'un':
tokenized = '\n'.join(tokens) tokenized = ' '.join(tokens)
out_filename = '%s.%s.txt' % (out_prefix, language) out_filename = '%s.%s.txt' % (out_prefix, language)
if out_filename in out_files: if out_filename in out_files:
out_file = out_files[out_filename] out_file = out_files[out_filename]