from wordfreq import tokenize, word_frequency
import pytest


def test_tokens():
    assert tokenize('감사합니다', 'ko') == ['감사', '합니다']


def test_combination():
    gamsa_freq = word_frequency('감사', 'ko')
    habnida_freq = word_frequency('합니다', 'ko')

    assert word_frequency('감사감사', 'ko') == pytest.approx(gamsa_freq / 2)
    assert (
        1.0 / word_frequency('감사합니다', 'ko') ==
        pytest.approx(1.0 / gamsa_freq + 1.0 / habnida_freq)
    )