mirror of
https://github.com/rspeer/wordfreq.git
synced 2024-12-24 09:51:38 +00:00
parent
a82dfedeba
commit
4b722a1f79
@ -198,12 +198,12 @@ def load_all_data(source_dir=None, filename=None, do_it_anyway=False):
|
||||
save_wordlist_to_db(conn, 'twitter-stems', lang, twitter_stems_wordlist[lang])
|
||||
|
||||
logger.info("Loading unstemmed Twitter corpus.")
|
||||
twitter_stems_wordlist = read_multilingual_csv(
|
||||
twitter_surface_wordlist = read_multilingual_csv(
|
||||
os.path.join(source_dir, 'luminoso', 'twitter-surfaces-2014.csv')
|
||||
)
|
||||
for lang in twitter_stems_wordlist:
|
||||
for lang in twitter_surface_wordlist:
|
||||
logger.info("\tLanguage: %s" % lang)
|
||||
save_wordlist_to_db(conn, 'twitter-surfaces', lang, twitter_stems_wordlist[lang])
|
||||
save_wordlist_to_db(conn, 'twitter-surfaces', lang, twitter_surface_wordlist[lang])
|
||||
|
||||
logger.info("Done loading.")
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user