iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-23 17:31:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

removed intermediate lists

Browse Source
This commit is contained in:
Joshua Chin 2015-07-07 14:57:45 -04:00
parent 4b49b1a547
commit 5342ea3033
1 changed files with 3 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
wordfreq/mecab.py
View File

@ -14,8 +14,6 @@ def mecab_tokenize(text):
contains the same table that the command-line version of MeCab would output.
We find the tokens in the first column of this table.
"""
parsed_str = MECAB_ANALYZER.parse(text.strip())
lines = [line for line in parsed_str.split('\n')
return [line.split('\t')[0]
for line in MECAB_ANALYZER.parse(text.strip()).split('\n')
if line != '' and line != 'EOS']
tokens = [line.split('\t')[0] for line in lines]
return tokens