Allow filler.py to use the external config file
Check in sample config file
This commit is contained in:
parent
87a4add95f
commit
2cc64919fd
11
config.py
Normal file
11
config.py
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
dbpath = "Database.db" # Path to the database file
|
||||||
|
|
||||||
|
user = "User Name" # Your twitter handle (without the @)
|
||||||
|
|
||||||
|
''' Details about your app '''
|
||||||
|
|
||||||
|
cke = "Consumer Key"
|
||||||
|
cse = "Consumer Secret"
|
||||||
|
ato = "Access Token"
|
||||||
|
ase = "Access Secret"
|
||||||
|
|
75
filler.py
75
filler.py
|
@ -1,59 +1,54 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
import html.parser, sqlite3, time, tweepy, os
|
import config, html.parser, sqlite3, time, tweepy, os
|
||||||
|
|
||||||
user = "Username"
|
def fill(user=config.user,dbpath=config.dbpath):
|
||||||
cke = "Consumer Key"
|
search = "from:" + user
|
||||||
cse = "Consumer Secret"
|
|
||||||
ato = "Access Token"
|
|
||||||
ase = "Access Secret"
|
|
||||||
|
|
||||||
search = "from:" + user
|
conn = sqlite3.connect(dbpath)
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
database_filename = "Database.db"
|
try:
|
||||||
|
cur.execute("SELECT max(tweet_id) FROM tweets")
|
||||||
|
except:
|
||||||
|
raise IOError("Database file " + dbpath + " does not exist or is in the wrong format. Please create it before trying to populate it.")
|
||||||
|
|
||||||
sql_conn = sqlite3.connect(database_filename)
|
try:
|
||||||
cur = sql_conn.cursor()
|
savepoint = int(cur.fetchone()[0])
|
||||||
|
except:
|
||||||
|
savepoint = 0
|
||||||
|
|
||||||
try:
|
last = savepoint
|
||||||
cur.execute("SELECT max(tweet_id) FROM tweets")
|
|
||||||
except:
|
|
||||||
print("Please create a database using ./makedb.py or ./csvdb.py before trying to populate it.")
|
|
||||||
|
|
||||||
try:
|
auth = tweepy.OAuthHandler(config.cke, config.cse)
|
||||||
savepoint = int(cur.fetchone()[0])
|
auth.set_access_token(config.ato, config.ase)
|
||||||
except:
|
api = tweepy.API(auth)
|
||||||
savepoint = 0
|
|
||||||
|
|
||||||
auth = tweepy.OAuthHandler(cke, cse)
|
timelineIterator = list(tweepy.Cursor(api.search, q=search, since_id=savepoint).items())
|
||||||
auth.set_access_token(ato, ase)
|
|
||||||
api = tweepy.API(auth)
|
|
||||||
|
|
||||||
timelineIterator = list(tweepy.Cursor(api.search, q=search, since_id=savepoint).items())
|
timeline = []
|
||||||
|
|
||||||
timeline = []
|
for status in timelineIterator:
|
||||||
|
timeline.append(status)
|
||||||
|
|
||||||
for status in timelineIterator:
|
timeline.reverse()
|
||||||
timeline.append(status)
|
|
||||||
|
|
||||||
timeline.reverse()
|
tw_counter = 0
|
||||||
|
|
||||||
tw_counter = 0
|
for status in timeline:
|
||||||
|
timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000"
|
||||||
|
text = html.parser.HTMLParser().unescape(status.text).replace("'", "''")
|
||||||
|
|
||||||
for status in timeline:
|
cur.execute("INSERT INTO tweets('tweet_id','timestamp','text') VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')")
|
||||||
print("(%(date)s) %(name)s: %(message)s\n" % \
|
last = status.id
|
||||||
{ "date" : status.created_at,
|
tw_counter = tw_counter + 1
|
||||||
"name" : status.author.screen_name.encode('utf-8'),
|
|
||||||
"message" : status.text.encode('utf-8') })
|
|
||||||
|
|
||||||
timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000"
|
conn.commit()
|
||||||
text = html.parser.HTMLParser().unescape(status.text).replace("'", "''")
|
conn.close()
|
||||||
|
|
||||||
cur.execute("INSERT INTO tweets('tweet_id','timestamp','text') VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')")
|
return tw_counter, last, savepoint
|
||||||
tw_counter = tw_counter + 1
|
|
||||||
|
|
||||||
sql_conn.commit()
|
if __name__ == "__main__":
|
||||||
sql_conn.close()
|
count, last, first = fill()
|
||||||
|
print("Stored %i tweets after %i until %i." % (count, first, last))
|
||||||
print("Finished. %d Tweets stored" % (tw_counter))
|
|
||||||
|
|
Loading…
Reference in a new issue