mirror of
https://github.com/rspeer/wordfreq.git
synced 2024-12-23 09:21:37 +00:00
64 lines
2.5 KiB
Python
64 lines
2.5 KiB
Python
from nose.tools import eq_, assert_almost_equal
|
|
from wordfreq import tokenize, simple_tokenize, word_frequency
|
|
|
|
|
|
def test_tokens():
|
|
eq_(tokenize('おはようございます', 'ja'),
|
|
['おはよう', 'ござい', 'ます'])
|
|
|
|
|
|
def test_simple_tokenize():
|
|
# When Japanese is run through simple_tokenize -- either because it's
|
|
# tagged with the wrong language, or because we want to pass through
|
|
# Japanese text without getting MeCab involved -- it will be split at
|
|
# boundaries between Japanese and non-Japanese scripts, but all Japanese
|
|
# scripts will be stuck together. Here the switch between hiragana
|
|
# (ひらがな) and katakana (カタカナ) is not a boundary, but the switch
|
|
# between katakana and romaji is.
|
|
#
|
|
# We used to try to infer word boundaries between hiragana and katakana,
|
|
# but this leads to edge cases that are unsolvable without a dictionary.
|
|
ja_text = 'ひらがなカタカナromaji'
|
|
eq_(
|
|
simple_tokenize(ja_text),
|
|
['ひらがなカタカナ', 'romaji']
|
|
)
|
|
|
|
# An example that would be multiple tokens if tokenized as 'ja' via MeCab,
|
|
# but sticks together in simple_tokenize
|
|
eq_(simple_tokenize('おはようございます'), ['おはようございます'])
|
|
|
|
# Names that use the weird possessive marker ヶ, which is technically a
|
|
# katakana even though it's being used like a kanji, stay together as one
|
|
# token
|
|
eq_(simple_tokenize("犬ヶ島"), ["犬ヶ島"])
|
|
|
|
# The word in ConceptNet that made me notice that simple_tokenize used
|
|
# to have a problem with the character 々
|
|
eq_(simple_tokenize("晴々しい"), ["晴々しい"])
|
|
|
|
# Explicit word separators are still token boundaries, such as the dot
|
|
# between "toner" and "cartridge" in "toner cartridge"
|
|
eq_(simple_tokenize("トナー・カートリッジ"), ["トナー", "カートリッジ"])
|
|
|
|
# This word has multiple weird characters that aren't quite kanji in it,
|
|
# and is in the dictionary
|
|
eq_(simple_tokenize("見ヶ〆料"), ["見ヶ〆料"])
|
|
|
|
|
|
|
|
def test_combination():
|
|
ohayou_freq = word_frequency('おはよう', 'ja')
|
|
gozai_freq = word_frequency('ござい', 'ja')
|
|
masu_freq = word_frequency('ます', 'ja')
|
|
|
|
assert_almost_equal(
|
|
word_frequency('おはようおはよう', 'ja'),
|
|
ohayou_freq / 2
|
|
)
|
|
assert_almost_equal(
|
|
1.0 / word_frequency('おはようございます', 'ja'),
|
|
1.0 / ohayou_freq + 1.0 / gozai_freq + 1.0 / masu_freq
|
|
)
|
|
|