diff --git a/dbtools/__init__.py b/dbtools/__init__.py new file mode 100644 index 0000000..a3c1f30 --- /dev/null +++ b/dbtools/__init__.py @@ -0,0 +1,79 @@ +import setuptools +import sqlite3, pymysql, pymysql.cursors + +SQLITE = 0 +MYSQL = 1 +MARIADB = MYSQL + +MIN = 0 +MAX = 1 + +class dbObject: + +# --------------------------------------------- Initialization ------------------------------------------------- + + def initMySQL(self, host, port, user, pwd, db): + self.conn = pymysql.connect(host = host, port = port, user = user, password = pwd, db = db, charset = "utf8mb4", cursorclass = pymysql.cursors.DictCursor) + self.cur = self.conn.cursor() + self.dbtype = MYSQL + self.host = host + self.port = port + self.user = user + self.pwd = pwd + self.db = db + + def initSQLite(self, path): + self.conn = sqlite3.connect(path) + self.cur = self.conn.cursor() + self.dbtype = SQLITE + self.path = path + + def __init__(self, dbtype = SQLITE, path = None, host = None, port = None, user = None, pwd = None, db = None): + + if dbtype == SQLITE: + self.initSQLite(path or 'Database.db') + + elif dbtype == MYSQL: + self.initMySQL(host or 'localhost', port or 3306, user, pwd, db) + + else: + raise ValueError("Unknown database type %s." % str(dbtype)) + +# ---------------------------------------------- No more initialization ---------------------------------------- + + def closeConnection(self): + return self.conn.close() + + def commit(self): + return self.conn.commit() + + def executeQuery(self, query): + return self.cur.execute(query) + + def getAll(self): + return self.cur.fetchall() + + def getNext(self): + return self.cur.fetchone() + + def isInitialized(self): + try: + self.executeQuery("SELECT * FROM messages") + return True + except: + return False + + def getLatestMessage(db): + db.executeQuery("SELECT max(id) FROM messages") + try: + return int(db.getNext()[0]) + except: + return 0 + +def dbHelper(): + if setuptools.dbtype() == SQLITE: + return dbObject(dbtype=SQLITE, path=setuptools.dbpath()) + elif setuptools.dbtype() == MYSQL: + return dbObject(dbtype=MYSQL, host=setuptools.dbhost(), user=setuptools.dbuser(), pwd=setuptools.dbpass(), db=setuptools.dbname()) + else: + raise setuptools.SetupException() diff --git a/messages.py b/messages.py new file mode 100755 index 0000000..9474816 --- /dev/null +++ b/messages.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 + +from BeautifulSoup import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.common.keys import Keys +from selenium.common.exceptions import NoSuchElementException +import urllib.request, urllib.error, urllib.parse, time, os +import setuptools + +def status(driver): + if "/main/login.php" not in driver.page_source: + return True + else: + return False + +def loadPage(url,period=5,init=False, driver=driver): + if not (init or status(driver)): + login() + driver.get(url) + time.sleep(period) + +def loginHandler(user = setuptools.user, password = setuptools.password, driver = driver): + loadPage("http://www.planetromeo.com/",10,True,driver) + + curfield = driver.find_element_by_name("username") + curfield.send_keys(user) + + curfield = driver.find_element_by_name("passwort") + curfield.send_keys(password) + + curfield.send_keys(Keys.RETURN) + time.sleep(10) + + return status() + +class LoginError(Exception): + pass + +def login(): + if not (status() or loginHandler()): + raise LoginError("Login failed.") + return True + +def messageID(url): + return url.split("?")[1] + +def messageHandler(mid, driver): + loadPage("https://www.planetromeo.com/msg/?id=" + mid, driver=driver) + + try: + links = driver.find_elements_by_partial_link_text('pix/popup.php/') + for link in links: + phototools.processURL(link.get_attribute('href'),vid) + except NoSuchElementException as e: + pass + +def pageHandler(driver): + webpage = driver.page_source + links = BeautifulSoup(webpage).findAll('a') + count = 0 + + for l in links: + url = l['href'] + if "/msg/?id=" in url: + count += 1 + mid = messageID(url) + if mid <= dbtools.getLatestMessage() + return False + messageHandler(mid, driver) + + if count == 0: + return False + + return True + +def siteHandler(p = 0, driver = driver): + loadPage("https://www.planetromeo.com/mitglieder/messages/uebersicht.php?seite=" + str(p), driver=driver) + if pageHandler(driver): + siteHandler(p+1, driver) + +if __name__ == "__main__": + driver = webdriver.Firefox() + if login(driver): + siteHandler(driver=driver) + + print("KTHXBAI") + + driver.close() diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..f9c4e6e --- /dev/null +++ b/setup.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +import configparser, os.path, tweepy, dbtools, getpass + +if os.path.isfile("config.cfg"): + print("config.cfg already exists. Please remove it before running this script.") + exit(1) + +config = configparser.RawConfigParser() + +config.add_section('Database') + +print('''Romeotools will use a database for certain tasks. You can use a file or a MySQL database for this purpose. + +If you wish to use a MySQL database, you will need the credentials for it. If you don't know what any of that means, stick with the default value and just press Enter. +''') + +dbtype = input("Database type: %i (file), %i (MySQL) [%i]: " % (dbtools.SQLITE, dbtools.MYSQL, dbtools.SQLITE)) +print() + +try: + dbtype = int(dbtype) +except: + pass + +if dbtype == dbtools.MYSQL: + dbhost = input("MySQL host [localhost]: ") or "localhost" + dbuser = input("MySQL username [twitools]: ") or "twitools" + dbpass = getpass.getpass("MySQL password (not echoed!): ") + dbname = input("MySQL database name [twitools]: ") or "twitools" + print() + + config.set('Database', 'type', dbtype) + config.set('Database', 'host', dbhost) + config.set('Database', 'user', dbuser) + config.set('Database', 'pass', dbpass) + config.set('Database', 'name', dbname) + +else: + dbtype = dbtools.SQLITE + dbpath = input("Name of the database file [Database.db]: ") or "Database.db" + print() + + config.set('Database', 'type', dbtype) + config.set('Database', 'path', dbpath) + +if dbtype == dbtools.MYSQL: + db = dbtools.dbObject(dbtype=dbtype, host=dbhost, user=dbuser, pwd=dbpass, db=dbname) +else: + db = dbtools.dbObject(dbtype=dbtype, path=dbpath) +if not db.isInitialized(): + db.executeQuery("CREATE TABLE messages(`id` INTEGER NOT NULL, `text` TEXT, `sender_id` VARCHAR(60), `recipient_id` VARCHAR(60), `created_at` TEXT, PRIMARY KEY(id));") + db.commit() + +db.closeConnection() + +config.add_section("Romeo") + +print('''In the next step, we'll get you connected to PlanetRomeo. For this, we +will need your username and password. Please note that these will be stored on +your hard disk in plain text. Sadly PlanetRomeo doesn't offer a better way for +third party applications to authenticate... +''') + +unam = input("Username: ") +pwrd = getpass.getpass("Password (not echoed back!): ") +print() + +config.set("Romeo", "user", unam) +config.set("Romeo", "pass", pwrd) + +print("Seems like everything worked out fine. Let's write that config file...") + +with open('config.cfg', 'wt') as cfg: + config.write(cfg) + +print("We're all done. You can now use Twitools. Have fun!") diff --git a/setuptools/__init__.py b/setuptools/__init__.py new file mode 100644 index 0000000..23dcf17 --- /dev/null +++ b/setuptools/__init__.py @@ -0,0 +1,121 @@ +import configparser, csv, datetime, html.parser, itertools, os, sqlite3, sys + +class SetupException(Exception): + def __str__(self): + return "Seems like config.cfg has not been created yet or contains serious errors. Run setup.py to create it." + +def getSetting(section, setting, path = "config.cfg"): + config = configparser.RawConfigParser() + config.read(path) + return config.get(section, setting) + +def dbtype(): + try: + return int(getSetting("Database", "type")) + except: + return 0 # for SQLite3 + +### Must only be called AFTER dbtype()! ### + +def dbhost(): + try: + return getSetting("Database", "host") + except: + raise SetupException() + +def dbuser(): + try: + return getSetting("Database", "user") + except: + raise SetupException() + +def dbpass(): + try: + return getSetting("Database", "pass") + except: + raise SetupException() + +def dbname(): + try: + return getSetting("Database", "name") + except: + raise SetupException() + +def dbpath(): + try: + return getSetting("Database", "path") + except: + return SetupException() + +### + +def user(): + try: + return getSetting("Romeo", "user") + except: + raise SetupException() + +def password(): + try: + return getSetting("Romeo", "pass") + except: + raise SetupException() + +def dbCheck(db, create = False): + if (not create and dbInitialized(db)) or (create and not dbInitialized(db)): + return True + if create: + raise ValueError("Provided database file " + db.path + " is already initialized. Remove it manually before trying to recreate it.") + raise ValueError("Provided database file " + db.path + " is not initialized. Create it using makedb.py or csvdb.py") + + +def dbHelper(path, create = False): + db = dbObject(path) + dbCheck(db, create) + return db + +def dbInitialized(db): + return db.isInitialized() + +def fileExists(path): + return os.path.isfile(path) + +def getDate(date): + try: + return datetime.datetime.strptime(date, '%Y-%m-%d') + except ValueError: + raise ValueError("Dates must be in YYYY-MM-DD format.") + +def paginate(iterable, page_size): + while True: + i1, i2 = itertools.tee(iterable) + iterable, page = (itertools.islice(i1, page_size, None), list(itertools.islice(i2, page_size))) + if len(page) == 0: + break + yield page + +def parseArgs(argv): + args = [] + path = None + nextpath = False + + for a in argv[1:]: + if nextpath: + path = a + nextpath = False + elif a == "-f": + if path != None: + raise ValueError("You can only pass one database file.") + nextpath = True + else: + args += [a] + + return args, path + +def printCSV(inlist): + writer = csv.writer(sys.stdout) + writer.writerows(inlist) + +def unescapeText(text): + return html.parser.HTMLParser().unescape(text).replace("'","''") +