iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-23 17:31:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

updated tests for emoji splitting

Browse Source
This commit is contained in:
Joshua Chin 2015-06-25 11:25:51 -04:00
parent 35a80e5f50
commit 3bcb3e84a1
1 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
tests/test.py
View File

@ -1,6 +1,6 @@
from wordfreq import (
word_frequency, available_languages, cB_to_freq, iter_wordlist,
top_n_list, random_words, random_ascii_words
top_n_list, random_words, random_ascii_words, tokenize
)
from nose.tools import (
eq_, assert_almost_equal, assert_greater, assert_less, raises
@ -84,12 +84,16 @@ def test_failed_cB_conversion():
def test_tokenization():
# We preserve apostrophes within words, so "can't" is a single word in the
# data, while the fake word "plan't" can't be found.
assert_greater(word_frequency("can't", 'en'), 0)
eq_(word_frequency("plan't", 'en'), 0)
eq_(tokenize("can't", 'en'), ["can't"])
eq_(tokenize("plan't", 'en'), ["plan't"])
eq_(tokenize('😂test', 'en'), ['😂', 'test'])
# We do split at other punctuation, causing the word-combining rule to
# apply.
assert_greater(word_frequency("can.t", 'en'), 0)
eq_(tokenize("can.t", 'en'), ['can', 't'])
def test_phrase_freq():
plant = word_frequency("plan.t", 'en')
assert_greater(plant, 0)
assert_less(plant, word_frequency('plan', 'en'))