mirror of
https://github.com/rspeer/wordfreq.git
synced 2024-12-23 17:31:41 +00:00
parent
3674d35501
commit
4ec128adae
@ -123,7 +123,6 @@ def google_books_deps(dirname_in):
|
|||||||
|
|
||||||
def twitter_deps(input_filename, slice_prefix, combined_prefix, slices,
|
def twitter_deps(input_filename, slice_prefix, combined_prefix, slices,
|
||||||
languages):
|
languages):
|
||||||
|
|
||||||
lines = []
|
lines = []
|
||||||
|
|
||||||
slice_files = ['{prefix}.part{num:0>2d}'.format(prefix=slice_prefix,
|
slice_files = ['{prefix}.part{num:0>2d}'.format(prefix=slice_prefix,
|
||||||
|
@ -11,7 +11,10 @@ CLD2_BAD_CHAR_RANGE = "[%s]" % "".join(
|
|||||||
'\x0e-\x1f',
|
'\x0e-\x1f',
|
||||||
'\x7f-\x9f',
|
'\x7f-\x9f',
|
||||||
'\ud800-\udfff',
|
'\ud800-\udfff',
|
||||||
'\ufdd0-\ufdef'
|
'\ufdd0-\ufdef',
|
||||||
|
'\N{HANGUL FILLER}',
|
||||||
|
'\N{HANGUL CHOSEONG FILLER}',
|
||||||
|
'\N{HANGUL JUNGSEONG FILLER}'
|
||||||
] +
|
] +
|
||||||
[chr(65534+65536*x+y) for x in range(17) for y in range(2)]
|
[chr(65534+65536*x+y) for x in range(17) for y in range(2)]
|
||||||
)
|
)
|
||||||
|
Loading…
Reference in New Issue
Block a user