iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-23 09:21:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #66 from LuminosoInsight/update-msgpack-call

Browse Source 
Update msgpack parameter
This commit is contained in:
Lance Nathan 2019-02-05 11:17:07 -05:00 committed by GitHub
commit 96b9808550
4 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/make_chinese_mapping.py
View File

@ -36,7 +36,7 @@ def make_hanzi_converter(table_in, msgpack_out):
if chr(codept) != char:
table[codept] = char
with gzip.open(msgpack_out, 'wb') as outfile:
msgpack.dump(table, outfile, encoding='utf-8')
msgpack.dump(table, outfile, raw=False)
def build():

4
setup.py
View File

@ -28,14 +28,14 @@ README_contents = open(os.path.join(current_dir, 'README.md'),
encoding='utf-8').read()
doclines = README_contents.split("\n")
dependencies = [
'msgpack', 'langcodes >= 1.4.1', 'regex == 2018.02.21'
'msgpack', 'langcodes >= 1.4.1', 'regex >= 2017.07.11, <= 2018.02.21'
]
if sys.version_info < (3, 4):
dependencies.append('pathlib')
setup(
name="wordfreq",
version='2.2.0',
version='2.2.1',
maintainer='Robyn Speer',
maintainer_email='rspeer@luminoso.com',
url='http://github.com/LuminosoInsight/wordfreq/',

2
wordfreq/__init__.py
View File

@ -74,7 +74,7 @@ def read_cBpack(filename):
]
"""
with gzip.open(filename, 'rb') as infile:
data = msgpack.load(infile, encoding='utf-8')
data = msgpack.load(infile, raw=False)
header = data[0]
if (
not isinstance(header, dict) or header.get('format') != 'cB'

2
wordfreq/chinese.py
View File

@ -6,7 +6,7 @@ import gzip
DICT_FILENAME = resource_filename('wordfreq', 'data/jieba_zh.txt')
ORIG_DICT_FILENAME = resource_filename('wordfreq', 'data/jieba_zh_orig.txt')
SIMP_MAP_FILENAME = resource_filename('wordfreq', 'data/_chinese_mapping.msgpack.gz')
SIMPLIFIED_MAP = msgpack.load(gzip.open(SIMP_MAP_FILENAME), encoding='utf-8')
SIMPLIFIED_MAP = msgpack.load(gzip.open(SIMP_MAP_FILENAME), raw=False)
jieba_tokenizer = None
jieba_orig_tokenizer = None