iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-23 17:31:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

also NFKC-normalize Japanese input

Browse Source
This commit is contained in:
Rob Speer 2015-08-24 18:13:03 -04:00
parent 1d055edc1c
commit 554455699d
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
wordfreq/mecab.py
View File

@ -1,4 +1,5 @@
import MeCab import MeCab
import unicodedata
# Instantiate the MeCab analyzer, which the mecab-python3 interface calls a # Instantiate the MeCab analyzer, which the mecab-python3 interface calls a
@ -14,6 +15,7 @@ def mecab_tokenize(text):
contains the same table that the command-line version of MeCab would output. contains the same table that the command-line version of MeCab would output.
We find the tokens in the first column of this table. We find the tokens in the first column of this table.
""" """
text = unicodedata.normalize('NFKC', text.strip())
return [line.split('\t')[0] return [line.split('\t')[0]
for line in MECAB_ANALYZER.parse(text.strip()).split('\n') for line in MECAB_ANALYZER.parse(text).split('\n')
if line != '' and line != 'EOS'] if line != '' and line != 'EOS']