added arabic tests

Former-commit-id: f83d31a357
2024-12-23 17:31:41 +00:00 · 2015-07-07 15:10:59 -04:00 · 2015-07-07 15:10:59 -04:00 · b3a008f992
commit b3a008f992
parent 21c809416d
1 changed files with 11 additions and 0 deletions
--- a/tests/test.py
+++ b/tests/test.py
@ -137,3 +137,14 @@ def test_not_really_random():
@raises(ValueError)
 def test_not_enough_ascii():
    random_ascii_words(lang='zh')
+
+def test_ar():
+    eq_(
+        tokenize('متــــــــعب', 'ar'),
+        ['متعب']
+    )
+
+    eq_(
+        tokenize('حَرَكَات', 'ar'),
+        ['حركات']
+    )