iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-23 09:21:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

change math for INFERRED_SPACE_FACTOR to not overflow

Browse Source
This commit is contained in:
Robyn Speer 2021-02-18 14:44:39 -05:00
parent 7318f58df9
commit 75a56b68fb
2 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
tests/test_chinese.py
View File

@ -77,3 +77,11 @@ def test_alternate_codes():
# Separate codes for Mandarin and Cantonese # Separate codes for Mandarin and Cantonese
assert tokenize('谢谢谢谢', 'cmn') == tokens assert tokenize('谢谢谢谢', 'cmn') == tokens
assert tokenize('谢谢谢谢', 'yue') == tokens assert tokenize('谢谢谢谢', 'yue') == tokens
def test_unreasonably_long():
# This crashed earlier versions of wordfreq
lots_of_ls = 'l' * 800
assert word_frequency(lots_of_ls, 'zh') < 1e-300
assert zipf_frequency(lots_of_ls, 'zh') == 0.

2
wordfreq/__init__.py
View File

@ -263,7 +263,7 @@ def _word_frequency(word, lang, wordlist, minimum):
# If we used the Jieba tokenizer, we could tokenize anything to match # If we used the Jieba tokenizer, we could tokenize anything to match
# our wordlist, even nonsense. To counteract this, we multiply by a # our wordlist, even nonsense. To counteract this, we multiply by a
# probability for each word break that was inferred. # probability for each word break that was inferred.
freq /= INFERRED_SPACE_FACTOR ** (len(tokens) - 1) freq *= INFERRED_SPACE_FACTOR ** -(len(tokens) - 1)
# All our frequency data is only precise to within 1% anyway, so round # All our frequency data is only precise to within 1% anyway, so round
# it to 3 significant digits # it to 3 significant digits