iskm/wordfreq
1
0
Fork 0
mirror of https://github.com/rspeer/wordfreq.git synced 2024-12-23 17:31:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

removed TOKENIZE_TWITTER

Browse Source 
Former-commit-id: 449a656edd
This commit is contained in:
Joshua Chin 2015-07-17 14:43:14 -04:00
parent d0df4cc9a4
commit 354f09ec24
1 changed files with 4 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
wordfreq_builder/wordfreq_builder/ninja.py
View File

@ -119,17 +119,15 @@ def google_books_deps(dirname_in):
return lines
def twitter_deps(input_filename, slice_prefix,
combined_prefix, slices, languages):
def twitter_deps(input_filename, slice_prefix, combined_prefix, slices, languages):
lines = []
slice_files = ['{prefix}.part{num:0>2d}'.format(prefix=slice_prefix, num=num)
for num in range(slices)]
# split the input into slices
add_dep(lines,
'split', input_filename, slice_files,
params={'prefix': '{}.part'.format(slice_prefix),
'slices': slices})
add_dep(lines, 'split', input_filename, slice_files,
params={'prefix': '{}.part'.format(slice_prefix), 'slices': slices})
for slicenum in range(slices):
slice_file = slice_files[slicenum]