From bb1716531a63f808f77545dfa0b62c4d599f5f5f Mon Sep 17 00:00:00 2001 From: Klaus-Uwe Mitterer Date: Tue, 7 Mar 2017 14:17:21 +0100 Subject: [PATCH] Move fillerfilter to filters.filler --- .gitignore | 2 ++ filler.py | 14 ++++++------- filters/filler.py.dist | 47 ++++++++++++++++++++++++++++++++++++++++++ setup.py | 5 ++++- 4 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 filters/filler.py.dist diff --git a/.gitignore b/.gitignore index 9b899b6..216763f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ __pycache__ config.cfg *.pyc .config/ +filters/*.py +media/ diff --git a/filler.py b/filler.py index d85f968..22aba1e 100755 --- a/filler.py +++ b/filler.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import argparse, dbtools, dbtools.fillerfilter, requests, setuptools, time, twitools +import argparse, dbtools, filters.filler, requests, setuptools, time, twitools def downloadMedia(url, tid, mid): remote = requests.get(url, stream=True) @@ -25,7 +25,7 @@ def getTweets(db=dbtools.dbHelper(), user=twitools.twObject().whoami(), two=twit timestamp = status.created_at.strftime('%Y-%m-%d %H:%M:%S') + " +0000" text = setuptools.unescapeText(status.text) - if dbtools.fillerfilter.tweetFilter(status): + if filters.filler.tweetFilter(status): try: db.executeQuery("INSERT INTO tweets(tweet_id,timestamp,text) VALUES(" + str(status.id) + ",'" + timestamp + "','" + text + "')") except: @@ -51,7 +51,7 @@ def getMessages(db=dbtools.dbHelper(), two=twitools.twObject()): new_out_messages = two.api.sent_direct_messages(since_id=savepoint, count=200, full_text=True, include_entities=True) for m in (new_messages + new_out_messages): - if dbtools.fillerfilter.messageFilter(m, True): + if filters.filler.messageFilter(m, True): try: db.executeQuery("INSERT INTO messages VALUES(%s, '%s', %s, %s, '%s')" % (m.id, setuptools.unescapeText(m.text), m.sender_id, m.recipient_id, m.created_at)) mcount += 1 @@ -79,13 +79,13 @@ def getFollowers(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False) return 0,0 for follower in new: - if follower not in current and dbtools.fillerfilter.followerFilter(follower, True): + if follower not in current and filters.filler.followerFilter(follower, True): db.executeQuery("INSERT INTO followers VALUES('%s', %i, 0)" % (str(follower), int(time.time()))) db.commit() gained += 1 for follower in current: - if follower not in new and dbtools.fillerfilter.followerFilter(follower, False): + if follower not in new and filters.filler.followerFilter(follower, False): db.executeQuery("UPDATE followers SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), str(follower))) db.commit() lost += 1 @@ -103,13 +103,13 @@ def getFollowing(db=dbtools.dbHelper(), two=twitools.twObject(), firstrun=False) return 0,0 for following in new: - if following not in current and dbtools.fillerfilter.followingFilter(following, True): + if following not in current and filters.filler.followingFilter(following, True): db.executeQuery("INSERT INTO following VALUES('%s', %i, 0)" % (str(following), int(time.time()))) db.commit() gained += 1 for following in current: - if following not in new and dbtools.fillerfilter.followingFilter(following, False): + if following not in new and filters.filler.followingFilter(following, False): db.executeQuery("UPDATE following SET `until` = %i WHERE `id` = '%s' AND `until` = 0" % (int(time.time()), str(following))) db.commit() lost += 1 diff --git a/filters/filler.py.dist b/filters/filler.py.dist new file mode 100644 index 0000000..5eadb9d --- /dev/null +++ b/filters/filler.py.dist @@ -0,0 +1,47 @@ +""" +This file allows you to add your own hooks to the filler.py script without +having to mess around with the code. +""" + +def tweetFilter(tweet): + """ + Code to be executed when a new tweet is found. + + :param tweet: tweepy.Status object for the new tweet + :return: True if the tweet should be stored in the database, else False + """ + + return True + +def messageFilter(message, incoming): + """ + Code to be executed when a new incoming or outgoing message is found. + + :param message: tweepy.DirectMessage object for the new message + :param incoming: True if the message is incoming (not sent by us), else False + :return: True if the message should be stored in the database, else False + """ + + return True + +def followerFilter(follower, current): + """ + Code to be executed when a follower is gained or lost. + + :param follower: Integer ID of the following account + :param current: True if they started following, False if they unfollowed + :return: True if the following/unfollowing action should be stored, else False + """ + + return True + +def followingFilter(following, current): + """ + Code to be executed when we start or stop following another account. + + :param following: Integer ID of the account we follow/stopped following + :param current: True if we started following, False if we unfollowed + :return: True if the following/unfollowing action should be stored, else False + """ + + return True diff --git a/setup.py b/setup.py index 355cb05..1945f6f 100755 --- a/setup.py +++ b/setup.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 -import configparser, os.path, tweepy, dbtools, getpass +import configparser, os.path, tweepy, dbtools, getpass, shutil os.makedirs("media", exist_ok=True) +if not os.path.isfile("filters/filler.py"): + shutil.copyfile("filters/filler.py.dist", "filters/filler.py") + if os.path.isfile("config.cfg"): print("config.cfg already exists. Please remove it before running this script.") exit(1)