Split up tools to setuptools and twitools. Testing pending.

This commit is contained in:
Klaus-Uwe Mitterer 2016-05-30 20:40:21 +02:00
parent 7e9e27e00b
commit 1c0b3da582
9 changed files with 49 additions and 96 deletions

View file

@ -1,8 +1,8 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import dbtools, tools import dbtools, setuptools, twitools
def fill(db=dbtools.dbObject(), user=tools.user(), two=tools.twObject()): def fill(db=dbtools.dbObject(), user=twitools.twObject().whoami(), two=twitools.twObject()):
query = "from:" + user query = "from:" + user
savepoint = db.getLatestTweet() savepoint = db.getLatestTweet()
last = savepoint last = savepoint
@ -13,7 +13,7 @@ def fill(db=dbtools.dbObject(), user=tools.user(), two=tools.twObject()):
for status in timeline: for status in timeline:
timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000" timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000"
text = tools.unescapeText(status.text) text = setuptools.unescapeText(status.text)
db.executeQuery("INSERT INTO tweets('tweet_id','timestamp','text') VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')") db.executeQuery("INSERT INTO tweets('tweet_id','timestamp','text') VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')")
db.commit() db.commit()
@ -28,3 +28,4 @@ def fill(db=dbtools.dbObject(), user=tools.user(), two=tools.twObject()):
if __name__ == "__main__": if __name__ == "__main__":
count, last, first = fill() count, last, first = fill()
print("Stored %i tweets after %i until %i." % (count, first, last)) print("Stored %i tweets after %i until %i." % (count, first, last))

View file

@ -1,14 +1,14 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import tools import twitools, setuptools
import os, time, tweepy import os, time, tweepy
def getFollowerIDs(two=tools.twObject()): def getFollowerIDs(two=twitools.twObject()):
''' Returns 5,000 follower IDs at most ''' ''' Returns 5,000 follower IDs at most '''
return two.api.followers_ids(screen_name=tools.user()) return two.api.followers_ids(screen_name=twitools.twObject().whoami())
def getNamesByIDs(fids=getFollowerIDs(), two=tools.twObject()): def getNamesByIDs(fids=getFollowerIDs(), two=twitools.twObject()):
for page in tools.paginate(fids, 100): for page in setuptools.paginate(fids, 100):
followers = two.api.lookup_users(user_ids=page) followers = two.api.lookup_users(user_ids=page)
for follower in followers: for follower in followers:
yield follower.screen_name yield follower.screen_name

View file

@ -1,7 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import tools import dbtools, setuptools
import sys, datetime import sys, datetime
def dateArgs(argv = sys.argv[1:]): def dateArgs(argv = sys.argv[1:]):
@ -30,10 +29,10 @@ def dateArgs(argv = sys.argv[1:]):
raise ValueError("Number of days for running average must be an integer.") raise ValueError("Number of days for running average must be an integer.")
mode = 0 mode = 0
elif mode == 1: elif mode == 1:
fr = tools.getDate(arg) fr = setuptools.getDate(arg)
mode = 0 mode = 0
else: else:
to = tools.getDate(arg) to = setuptools.getDate(arg)
mode = 0 mode = 0
if mode in (1, 2): if mode in (1, 2):
@ -92,9 +91,7 @@ def getHeaders(strings, av):
return [headers] return [headers]
def getTweetsByDate(strings = [], fr = None, to = None, av = 0, path = tools.dbpath(), headers = False): def getTweetsByDate(strings = [], fr = None, to = None, av = 0, db = dbtools.dbObject(), headers = False):
db = tools.dbHelper(path)
if fr == None: if fr == None:
fr = db.getFLDate() fr = db.getFLDate()
if to == None: if to == None:
@ -111,4 +108,4 @@ def getTweetsByDate(strings = [], fr = None, to = None, av = 0, path = tools.dbp
return cur return cur
if __name__ == "__main__": if __name__ == "__main__":
tools.printCSV(getTweetsByDate(*dateArgs(), headers = True)) setuptools.printCSV(getTweetsByDate(*dateArgs(), headers = True))

View file

@ -1,12 +1,10 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import tools import dbtools
import operator, re, sys import operator, re, sys
def getTweets(mode = "@", path = tools.dbpath()): def getTweets(mode = "@", db = dbtools.dbObject()):
db = tools.dbHelper(path)
handles = dict() handles = dict()
tweets = db.executeQuery("SELECT text FROM tweets") tweets = db.executeQuery("SELECT text FROM tweets")
@ -27,12 +25,12 @@ def getTweets(mode = "@", path = tools.dbpath()):
if __name__ == "__main__": if __name__ == "__main__":
mode = "@" mode = "@"
path = tools.dbpath()
if len(sys.argv) > 1: if len(sys.argv) > 1:
if len(sys.argv) > 3 or (len(sys.argv) == 3 and "-h" not in sys.argv): if len(sys.argv) > 3 or (len(sys.argv) == 3 and "-h" not in sys.argv):
raise ValueError("Invalid arguments passed.") raise ValueError("Invalid arguments passed.")
path = None
for arg in sys.argv[1:]: for arg in sys.argv[1:]:
if arg == "-h": if arg == "-h":
mode = "#" mode = "#"
@ -41,5 +39,5 @@ if __name__ == "__main__":
else: else:
path = arg path = arg
for handle, tweets in sorted(list(getTweets(mode,path).items()), key=operator.itemgetter(1), reverse=True): for handle, tweets in sorted(list(getTweets(mode,dbtools.dbObject(path=path)).items()), key=operator.itemgetter(1), reverse=True):
print(handle + "," + str(tweets)) print(handle + "," + str(tweets))

View file

@ -1,6 +1,6 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import tools import dbtools, setuptools
import sys, datetime import sys, datetime
@ -20,10 +20,10 @@ def dateArgs(argv = sys.argv[1:]):
strings += [arg] strings += [arg]
mode = 0 mode = 0
elif mode == 1: elif mode == 1:
fr = tools.getDate(arg) fr = setuptools.getDate(arg)
mode = 0 mode = 0
else: else:
to = tools.getDate(arg) to = setuptools.getDate(arg)
mode = 0 mode = 0
if mode in (1, 2): if mode in (1, 2):
@ -39,9 +39,7 @@ def queryBuilder(fr, to):
return "SELECT * FROM tweets WHERE SUBSTR(timestamp,0,11) >= '%s' AND SUBSTR(timestamp,0,11) <= '%s'" % (fr, to) return "SELECT * FROM tweets WHERE SUBSTR(timestamp,0,11) >= '%s' AND SUBSTR(timestamp,0,11) <= '%s'" % (fr, to)
def getDataByDate(fr, to, path = tools.dbpath(), headers = True): def getDataByDate(fr, to, db = dbtools.dbObject(), headers = True):
db = tools.dbHelper(path)
if fr == None: if fr == None:
fr = db.getFLDate() fr = db.getFLDate()
if to == None: if to == None:

View file

@ -1,12 +1,10 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import tools import dbtools
import sys import sys
def makeDB(path=tools.dbpath()): def makeDB(db=dbtools.dbObject()):
db = tools.dbHelper(path, create = True)
db.executeQuery("CREATE TABLE tweets(`tweet_id` INTEGER NOT NULL, `in_reply_to_status_id` TEXT, `in_reply_to_user_id` TEXT, `timestamp` TEXT, `source` TEXT, `text` TEXT, `retweeted_status_id` TEXT, `retweeted_status_user_id` TEXT, `retweeted_status_timestamp` TEXT, `expanded_urls` TEXT, PRIMARY KEY(tweet_id));") db.executeQuery("CREATE TABLE tweets(`tweet_id` INTEGER NOT NULL, `in_reply_to_status_id` TEXT, `in_reply_to_user_id` TEXT, `timestamp` TEXT, `source` TEXT, `text` TEXT, `retweeted_status_id` TEXT, `retweeted_status_user_id` TEXT, `retweeted_status_timestamp` TEXT, `expanded_urls` TEXT, PRIMARY KEY(tweet_id));")
db.commit() db.commit()
@ -16,6 +14,6 @@ if __name__ == "__main__":
if len(sys.argv) > 2: if len(sys.argv) > 2:
raise ValueError(sys.argv[0] + " only takes one argument, the path of the new database file.") raise ValueError(sys.argv[0] + " only takes one argument, the path of the new database file.")
try: try:
makeDB(sys.argv[1]) makeDB(dbtools.dbObject(path=sys.argv[1]))
except IndexError: except IndexError:
makeDB() makeDB()

View file

@ -40,64 +40,6 @@ def ase():
except: except:
raise SetupException() raise SetupException()
def user():
return twObject().whoami()
class dbObject:
def __init__(self, path=dbpath()):
self.conn = sqlite3.connect(path)
self.cur = self.conn.cursor()
self.path = path
def closeConnection(self):
return self.conn.close()
def commit(self):
return self.conn.commit()
def executeQuery(self, query):
return self.cur.execute(query)
def getNext(self):
return self.cur.fetchone()
def isInitialized(self):
try:
self.executeQuery("SELECT * FROM tweets")
return True
except:
return False
def getFLDate(self, val = 0):
if val == 0:
mode = "MIN"
else:
mode = "MAX"
return getDate(str(list(self.executeQuery("SELECT %s(SUBSTR(timestamp,0,11)) FROM tweets" % mode))[0][0]))
class twObject:
def __init__(self, cke = cke(), cse = cse(), ato = ato(), ase = ase()):
self.auth = tweepy.OAuthHandler(cke, cse)
self.auth.set_access_token(ato, ase)
self.api = tweepy.API(self.auth)
def delete(self, id):
self.api.destroy_status(id)
def search(self, query, savepoint = 0):
tweets = list(tweepy.Cursor(self.api.search, q=query, since_id=savepoint).items())
tweets.reverse()
return tweets
def whoami(self):
return self.auth.get_username()
def dbCheck(db, create = False): def dbCheck(db, create = False):
if (not create and dbInitialized(db)) or (create and not dbInitialized(db)): if (not create and dbInitialized(db)) or (create and not dbInitialized(db)):
return True return True

View file

@ -1,10 +1,9 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import tools import twitools
import tkinter, tkinter.messagebox, html.parser, os import tkinter, tkinter.messagebox, html.parser, os
two = tools.twObject() two = twitools.twObject()
top = tkinter.Tk() top = tkinter.Tk()
top.title("Tweet Deleter") top.title("Tweet Deleter")
scrollbar = tkinter.Scrollbar(top) scrollbar = tkinter.Scrollbar(top)
@ -41,7 +40,7 @@ def addStatus(id, text):
list.insert(0, element.encode("UTF-8")) list.insert(0, element.encode("UTF-8"))
def getTweets(): def getTweets():
query = "from:" + tools.user() query = "from:" + twitools.twObject().whoami()
try: try:
timeline = two.search(query, 0) timeline = two.search(query, 0)

20
twitools/__init__.py Normal file
View file

@ -0,0 +1,20 @@
import tweepy, setuptools
class twObject:
def __init__(self, cke = setuptools.cke(), cse = setuptools.cse(), ato = setuptools.ato(), ase = setuptools.ase()):
self.auth = tweepy.OAuthHandler(cke, cse)
self.auth.set_access_token(ato, ase)
self.api = tweepy.API(self.auth)
def delete(self, id):
self.api.destroy_status(id)
def search(self, query, savepoint = 0):
tweets = list(tweepy.Cursor(self.api.search, q=query, since_id=savepoint).items())
tweets.reverse()
return tweets
def whoami(self):
return self.auth.get_username()