From 701afaef45400fb3b5a8a32138a6e476573c5892 Mon Sep 17 00:00:00 2001 From: Klaus-Uwe Mitterer Date: Mon, 30 Jan 2017 03:04:51 +0100 Subject: [PATCH] Store followers/followings as user IDs rather than handles. Store handles in separate table. --- dbtools/__init__.py | 13 ++++++++++-- filler.py | 47 ++++++++++++++++++++++---------------------- twitools/__init__.py | 12 ++++++++--- 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/dbtools/__init__.py b/dbtools/__init__.py index e616238..95d508e 100644 --- a/dbtools/__init__.py +++ b/dbtools/__init__.py @@ -77,12 +77,12 @@ class dbObject: def getFollowers(db): db.executeQuery("SELECT id FROM followers WHERE `until` = 0;") for i in db.getAll(): - yield i[0] + yield int(i[0]) def getFollowing(db): db.executeQuery("SELECT id FROM following WHERE `until` = 0;") for i in db.getAll(): - yield i[0] + yield int(i[0]) def getLatestMessage(db): @@ -99,6 +99,15 @@ class dbObject: except: return 0 + def matchNameID(db, name, id): + db.executeQuery("SELECT COUNT(*) FROM names WHERE id = '%s' AND name = '%s' AND until = 0;" % (id, name)) + try: + if int(db.getNext()[0]) != 0: + return True + except: + pass + return False + def dbHelper(): if setuptools.dbtype() == SQLITE: return dbObject(dbtype=SQLITE, path=setuptools.dbpath()) diff --git a/filler.py b/filler.py index 298e61e..bab2d54 100755 --- a/filler.py +++ b/filler.py @@ -48,13 +48,11 @@ def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()): db.commit() - return mcount, savepoint or 0, db.getLatestMessage() + return mcount, savepoint or 0, db.getLatestMessage def getFollowers(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False): - """ Get handles of users we are following. :param db: Database object to be used. :param two: Twitter object to be used. :param firstrun: Must be set to True if the function is executed for the first time. Defaults to False. :return: Returns the number of gained and lost followings in a list (gained, lost). """ - current = list(db.getFollowers()) - new = list(twitools.getNamesByIDs(twitools.getFollowerIDs())) + new = list(twitools.getFollowerIDs()) gained = 0 lost = 0 @@ -64,31 +62,23 @@ def getFollowers(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False) for follower in new: if follower not in current: - db.executeQuery("INSERT INTO followers VALUES('%s', %i, 0)" % (follower, int(time.time()))) - print("New follower: %s" % follower) + db.executeQuery("INSERT INTO followers VALUES('%s', %i, 0)" % (str(follower), int(time.time()))) + db.commit() + print("New follower: %s" % (twitools.getNameByID(follower) if not firstrun else follower)) gained += 1 for follower in current: if follower not in new: - db.executeQuery("UPDATE followers SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), follower)) - print("Lost follower: %s" % follower) + db.executeQuery("UPDATE followers SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), str(follower))) + db.commit() + print("Lost follower: %s" % twitools.getNameByID(follower)) lost += 1 - db.commit() - return gained, lost def getFollowing(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False): - """ - Get handles of users we are following. - - :param db: Database object to be used. - :param two: Twitter object to be used. - :param firstrun: Must be set to True if the function is executed for the first time. Defaults to False. - :return: Returns the number of gained and lost followings in a list (gained, lost). - """ current = list(db.getFollowing()) - new = list(twitools.getNamesByIDs(twitools.getFollowingIDs())) + new = list(twitools.getFollowingIDs()) gained = 0 lost = 0 @@ -98,20 +88,29 @@ def getFollowing(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False) for following in new: if following not in current: - db.executeQuery("INSERT INTO following VALUES('%s', %i, 0)" % (following, int(time.time()))) - print("You started following: %s" % following) + db.executeQuery("INSERT INTO following VALUES('%s', %i, 0)" % (str(following), int(time.time()))) + db.commit() + print("You started following: %s" % (str(following) if not firstrun else following)) gained += 1 for following in current: if following not in new: - db.executeQuery("UPDATE following SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), following)) - print("You no longer follow: %s" % following) + db.executeQuery("UPDATE following SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), str(following))) + db.commit() + print("You no longer follow: %s" % twitools.getNameByID(following)) lost += 1 db.commit() return gained, lost +def getNames(db = dbtools.dbHelper(), two = twitools.twObject()): + for user in twitools.getNamesByIDs(list(set(list(db.getFollowing()) + list(db.getFollowers())))): + if not db.matchNameID(user["name"], user["id"]): + db.executeQuery("UPDATE names SET `until` = %i WHERE `id` = '%s' AND `name` = '%s';" % (int(time.time()), str(user["id"]), str(user["name"]))) + db.executeQuery("INSERT INTO names VALUES('%s', '%s', %i, 0)" % (str(user["id"]), str(user["name"]), int(time.time()))) + db.commit() + if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("-f", "--first", help="first run: ignore empty databases", action="store_true") @@ -125,3 +124,5 @@ if __name__ == "__main__": print("Gained %i followers, lost %i." % (gained, lost)) gained, lost = getFollowing(db, firstrun=args.first) print("Started following %i, stopped following %i." % (gained, lost)) + getNames(db) + print("Stored handles of following/followers.") diff --git a/twitools/__init__.py b/twitools/__init__.py index a5686f4..f69ef2b 100644 --- a/twitools/__init__.py +++ b/twitools/__init__.py @@ -26,13 +26,19 @@ class twObject: def getFollowerIDs(two=twObject()): ''' Returns 5,000 follower IDs at most ''' - return two.api.followers_ids(screen_name=twObject().whoami()) + for id in list(two.api.followers_ids(screen_name=twObject().whoami())): + yield int(id) def getFollowingIDs(two=twObject()): - return two.api.friends_ids(screen_name=twObject().whoami()) + for id in list(two.api.friends_ids(screen_name=twObject().whoami())): + yield int(id) + +def getNameByID(uid, two=twObject()): + return two.api.get_user(uid).screen_name def getNamesByIDs(fids=getFollowerIDs(), two=twObject()): for page in setuptools.paginate(fids, 100): followers = two.api.lookup_users(user_ids=page) for follower in followers: - yield follower.screen_name + yield {"id": follower.id, "name": follower.screen_name} +