iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-23 17:31:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fixed tokenize_twitter

Browse Source
This commit is contained in:
Joshua Chin 2015-07-17 16:37:47 -04:00
parent a44927e98e
commit f31f9a1bcd
1 changed files with 1 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
wordfreq_builder/wordfreq_builder/cli/tokenize_twitter.py
View File

@ -2,18 +2,12 @@ from wordfreq_builder.tokenizers import cld2_surface_tokenizer, tokenize_twitter
import argparse
def tokenize_twitter(in_filename, out_prefix):
tokenize_twitter(in_filename, out_prefix,
tokenizer=cld2_surface_tokenizer
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('filename', help='filename of input file containing one tweet per line')
parser.add_argument('outprefix', help='prefix of output filenames')
args = parser.parse_args()
tokenize_twitter(args.filename, args.outprefix)
tokenize_twitter(args.filename, args.outprefix, tokenizer=cld2_surface_tokenizer)
if __name__ == '__main__':