diff --git a/config.py b/config.py new file mode 100644 index 0000000..1bef04c --- /dev/null +++ b/config.py @@ -0,0 +1,11 @@ +dbpath = "Database.db" # Path to the database file + +user = "User Name" # Your twitter handle (without the @) + +''' Details about your app ''' + +cke = "Consumer Key" +cse = "Consumer Secret" +ato = "Access Token" +ase = "Access Secret" + diff --git a/filler.py b/filler.py index 1842ee3..ea58c22 100755 --- a/filler.py +++ b/filler.py @@ -1,59 +1,54 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -import html.parser, sqlite3, time, tweepy, os +import config, html.parser, sqlite3, time, tweepy, os -user = "Username" -cke = "Consumer Key" -cse = "Consumer Secret" -ato = "Access Token" -ase = "Access Secret" +def fill(user=config.user,dbpath=config.dbpath): + search = "from:" + user -search = "from:" + user + conn = sqlite3.connect(dbpath) + cur = conn.cursor() -database_filename = "Database.db" + try: + cur.execute("SELECT max(tweet_id) FROM tweets") + except: + raise IOError("Database file " + dbpath + " does not exist or is in the wrong format. Please create it before trying to populate it.") -sql_conn = sqlite3.connect(database_filename) -cur = sql_conn.cursor() + try: + savepoint = int(cur.fetchone()[0]) + except: + savepoint = 0 -try: - cur.execute("SELECT max(tweet_id) FROM tweets") -except: - print("Please create a database using ./makedb.py or ./csvdb.py before trying to populate it.") + last = savepoint -try: - savepoint = int(cur.fetchone()[0]) -except: - savepoint = 0 + auth = tweepy.OAuthHandler(config.cke, config.cse) + auth.set_access_token(config.ato, config.ase) + api = tweepy.API(auth) -auth = tweepy.OAuthHandler(cke, cse) -auth.set_access_token(ato, ase) -api = tweepy.API(auth) + timelineIterator = list(tweepy.Cursor(api.search, q=search, since_id=savepoint).items()) -timelineIterator = list(tweepy.Cursor(api.search, q=search, since_id=savepoint).items()) + timeline = [] -timeline = [] + for status in timelineIterator: + timeline.append(status) -for status in timelineIterator: - timeline.append(status) + timeline.reverse() -timeline.reverse() + tw_counter = 0 -tw_counter = 0 + for status in timeline: + timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000" + text = html.parser.HTMLParser().unescape(status.text).replace("'", "''") -for status in timeline: - print("(%(date)s) %(name)s: %(message)s\n" % \ - { "date" : status.created_at, - "name" : status.author.screen_name.encode('utf-8'), - "message" : status.text.encode('utf-8') }) + cur.execute("INSERT INTO tweets('tweet_id','timestamp','text') VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')") + last = status.id + tw_counter = tw_counter + 1 - timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000" - text = html.parser.HTMLParser().unescape(status.text).replace("'", "''") + conn.commit() + conn.close() - cur.execute("INSERT INTO tweets('tweet_id','timestamp','text') VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')") - tw_counter = tw_counter + 1 + return tw_counter, last, savepoint -sql_conn.commit() -sql_conn.close() - -print("Finished. %d Tweets stored" % (tw_counter)) +if __name__ == "__main__": + count, last, first = fill() + print("Stored %i tweets after %i until %i." % (count, first, last))