134 lines
4.8 KiB
Python
Executable file
134 lines
4.8 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
|
|
import httptools, argparse, dbtools, filters.filler, mediatools, requests, setuptools, time, twitools
|
|
|
|
def getTweets(db=dbtools.dbHelper(), user=twitools.twObject().whoami(), two=twitools.twObject()):
|
|
savepoint = db.getLatestTweet() + 1
|
|
last = savepoint
|
|
|
|
timeline = two.api.user_timeline(screen_name = user, since_id = savepoint, include_rts = True, tweet_mode = "extended")
|
|
|
|
tw_counter = 0
|
|
|
|
for status in timeline:
|
|
timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000"
|
|
text = setuptools.unescapeText(status.full_text)
|
|
|
|
if filters.filler.tweetFilter(status):
|
|
try:
|
|
db.executeQuery("INSERT INTO tweets(tweet_id,timestamp,text) VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')")
|
|
db.commit()
|
|
except Exception as e:
|
|
print(e)
|
|
print("Failed to insert %s into database." % str(status.id))
|
|
|
|
if 'media' in status.entities:
|
|
mid = 0
|
|
for m in status.entities['media']:
|
|
if "video" in m["expanded_url"]:
|
|
mediatools.videoHandler(m["expanded_url"], status.id, mid)
|
|
else:
|
|
mediatools.photoHandler(m['media_url'], status.id, mid)
|
|
mid += 1
|
|
|
|
last = status.id
|
|
tw_counter = tw_counter + 1
|
|
|
|
return tw_counter, last, savepoint
|
|
|
|
def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()):
|
|
mcount = 0
|
|
savepoint = db.getLatestMessage() + 1
|
|
new_messages = two.api.direct_messages(since_id=savepoint, count=200, full_text=True, include_entities=True)
|
|
new_out_messages = two.api.sent_direct_messages(since_id=savepoint, count=200, full_text=True, include_entities=True)
|
|
|
|
for m in (new_messages + new_out_messages):
|
|
if filters.filler.messageFilter(m, True):
|
|
try:
|
|
db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at))
|
|
db.commit()
|
|
mcount += 1
|
|
except:
|
|
pass
|
|
|
|
if 'media' in m.entities:
|
|
mid = 0
|
|
for med in m.entities['media']:
|
|
httptools.downloadMedia(med['media_url'], "m%i" % int(m.id), mid)
|
|
mid += 1
|
|
|
|
return mcount, savepoint or 0, db.getLatestMessage
|
|
|
|
def getFollowers(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False):
|
|
current = list(db.getFollowers())
|
|
new = list(twitools.getFollowerIDs())
|
|
gained = 0
|
|
lost = 0
|
|
|
|
if (len(current) == 0 or len(new) == 0) and not firstrun:
|
|
print("Something went wrong.")
|
|
return 0,0
|
|
|
|
for follower in new:
|
|
if follower not in current and filters.filler.followerFilter(follower, True):
|
|
db.executeQuery("INSERT INTO followers VALUES('%s', %i, 0)" % (str(follower), int(time.time())))
|
|
db.commit()
|
|
gained += 1
|
|
|
|
for follower in current:
|
|
if follower not in new and filters.filler.followerFilter(follower, False):
|
|
db.executeQuery("UPDATE followers SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), str(follower)))
|
|
db.commit()
|
|
lost += 1
|
|
|
|
return gained, lost
|
|
|
|
def getFollowing(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False):
|
|
current = list(db.getFollowing())
|
|
new = list(twitools.getFollowingIDs())
|
|
gained = 0
|
|
lost = 0
|
|
|
|
if (len(current) == 0 or len(new) == 0) and not firstrun:
|
|
print("Something went wrong.")
|
|
return 0,0
|
|
|
|
for following in new:
|
|
if following not in current and filters.filler.followingFilter(following, True):
|
|
db.executeQuery("INSERT INTO following VALUES('%s', %i, 0)" % (str(following), int(time.time())))
|
|
db.commit()
|
|
gained += 1
|
|
|
|
for following in current:
|
|
if following not in new and filters.filler.followingFilter(following, False):
|
|
db.executeQuery("UPDATE following SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), str(following)))
|
|
db.commit()
|
|
lost += 1
|
|
|
|
db.commit()
|
|
|
|
return gained, lost
|
|
|
|
def getNames(db = dbtools.dbHelper(), two = twitools.twObject()):
|
|
for user in twitools.getNamesByIDs(list(set(list(db.getFollowing()) + list(db.getFollowers())))):
|
|
if not db.matchNameID(user["name"], user["id"]):
|
|
db.executeQuery("UPDATE names SET `until` = %i WHERE `id` = '%s' AND `until` = 0;" % (int(time.time()), str(user["id"])))
|
|
db.executeQuery("INSERT INTO names VALUES('%s', '%s', %i, 0)" % (str(user["id"]), str(user["name"]), int(time.time())))
|
|
db.commit()
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("-f", "--first", help="first run: ignore empty databases", action="store_true")
|
|
args = parser.parse_args()
|
|
db = dbtools.dbHelper()
|
|
count, last, first = getTweets(db)
|
|
print("Stored %i tweets." % count)
|
|
count, last, first = getMessages(db)
|
|
print("Stored %i messages." % count)
|
|
gained, lost = getFollowers(db, firstrun=args.first)
|
|
print("Gained %i followers, lost %i." % (gained, lost))
|
|
gained, lost = getFollowing(db, firstrun=args.first)
|
|
print("Started following %i, stopped following %i." % (gained, lost))
|
|
getNames(db)
|
|
print("Stored handles of following/followers.")
|