iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-23 17:31:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

also NFKC-normalize Japanese input

Browse Source 
Former-commit-id: 554455699d
This commit is contained in:
Robyn Speer 2015-08-24 18:13:03 -04:00
parent 13096b26bd
commit 7bdfb74720
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
wordfreq/mecab.py
View File

@ -1,4 +1,5 @@
import MeCab import MeCab
import unicodedata
# Instantiate the MeCab analyzer, which the mecab-python3 interface calls a # Instantiate the MeCab analyzer, which the mecab-python3 interface calls a
@ -14,6 +15,7 @@ def mecab_tokenize(text):
contains the same table that the command-line version of MeCab would output. contains the same table that the command-line version of MeCab would output.
We find the tokens in the first column of this table. We find the tokens in the first column of this table.
""" """
text = unicodedata.normalize('NFKC', text.strip())
return [line.split('\t')[0] return [line.split('\t')[0]
for line in MECAB_ANALYZER.parse(text.strip()).split('\n') for line in MECAB_ANALYZER.parse(text).split('\n')
if line != '' and line != 'EOS'] if line != '' and line != 'EOS']