mirror of
https://github.com/rspeer/wordfreq.git
synced 2024-12-25 10:15:23 +00:00
91 lines
2.4 KiB
Python
91 lines
2.4 KiB
Python
import argparse
|
|
import unicodedata
|
|
from ftfy import chardata
|
|
import pathlib
|
|
from pkg_resources import resource_filename
|
|
|
|
|
|
DATA_PATH = pathlib.Path(resource_filename('wordfreq', 'data'))
|
|
|
|
|
|
def _emoji_char_class():
|
|
"""
|
|
Build a regex for emoji substitution. We create a regex character set
|
|
(like "[a-cv-z]") matching characters we consider emoji.
|
|
"""
|
|
emoji_file = DATA_PATH / 'emoji.txt'
|
|
|
|
def accept(c):
|
|
x = ord(c)
|
|
return chardata.CHAR_CLASS_STRING[x] == '3' and \
|
|
x >= 0x2600 and x != 0xfffd
|
|
|
|
with (DATA_PATH / 'emoji.txt').open(mode='w') as file:
|
|
file.write(func_to_regex(accept))
|
|
|
|
|
|
def _non_punct_class():
|
|
"""
|
|
Builds a regex that matches anything that is not a one of the following
|
|
classes:
|
|
- P: punctuation
|
|
- S: symbols
|
|
- Z: separators
|
|
- C: control characters
|
|
This will classify symbols, including emoji, as punctuation; callers that
|
|
want to treat emoji separately should filter them out first.
|
|
"""
|
|
non_punct_file = DATA_PATH / 'non_punct.txt'
|
|
|
|
out = func_to_regex(lambda c: unicodedata.category(c)[0] not in 'PSZC')
|
|
|
|
with non_punct_file.open(mode='w') as file:
|
|
file.write(out)
|
|
|
|
|
|
def _combining_mark_class():
|
|
"""
|
|
Builds a regex that matches anything that is a combining mark
|
|
"""
|
|
combining_mark_file = DATA_PATH / 'combining_mark.txt'
|
|
out = func_to_regex(lambda c: unicodedata.category(c)[0] == 'M')
|
|
|
|
with combining_mark_file.open(mode='w') as file:
|
|
file.write(out)
|
|
|
|
|
|
def func_to_regex(accept):
|
|
"""
|
|
Converts a function that accepts a single unicode character into a regex.
|
|
Unassigned unicode characters are treated like their neighbors.
|
|
"""
|
|
ranges = []
|
|
start = None
|
|
has_accepted = False
|
|
for x in range(0x110000):
|
|
c = chr(x)
|
|
|
|
if accept(c):
|
|
has_accepted = True
|
|
if start is None:
|
|
start = c
|
|
elif unicodedata.category(c) == 'Cn':
|
|
if start is None:
|
|
start = c
|
|
elif start is not None:
|
|
if has_accepted:
|
|
ranges.append('-'.join([start, chr(x-1)]))
|
|
has_accepted = False
|
|
start = None
|
|
else:
|
|
if has_accepted and start is not None:
|
|
ranges.append('-'.join([start, chr(x-1)]))
|
|
|
|
return '[%s]' % ''.join(ranges)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
_combining_mark_class()
|
|
_non_punct_class()
|
|
_emoji_char_class()
|