Check in stuff primarily copied over from twitools

dbtools/__init__.py

@@ -0,0 +1,79 @@
+import setuptools
+import sqlite3, pymysql, pymysql.cursors
+
+SQLITE = 0
+MYSQL = 1
+MARIADB = MYSQL
+
+MIN = 0
+MAX = 1
+
+class dbObject:
+
+# --------------------------------------------- Initialization -------------------------------------------------
+
+ def initMySQL(self, host, port, user, pwd, db):
+  self.conn = pymysql.connect(host = host, port = port, user = user, password = pwd, db = db, charset = "utf8mb4", cursorclass = pymysql.cursors.DictCursor)
+  self.cur = self.conn.cursor()
+  self.dbtype = MYSQL
+  self.host = host
+  self.port = port
+  self.user = user
+  self.pwd = pwd
+  self.db = db
+
+ def initSQLite(self, path):
+  self.conn = sqlite3.connect(path)
+  self.cur = self.conn.cursor()
+  self.dbtype = SQLITE
+  self.path = path
+
+ def __init__(self, dbtype = SQLITE, path = None, host = None, port = None, user = None, pwd = None, db = None):
+
+  if dbtype == SQLITE:
+   self.initSQLite(path or 'Database.db')
+
+  elif dbtype == MYSQL:
+   self.initMySQL(host or 'localhost', port or 3306, user, pwd, db)
+
+  else:
+   raise ValueError("Unknown database type %s." % str(dbtype))
+
+# ---------------------------------------------- No more initialization ----------------------------------------
+
+ def closeConnection(self):
+  return self.conn.close()
+
+ def commit(self):
+  return self.conn.commit()
+
+ def executeQuery(self, query):
+  return self.cur.execute(query)
+
+ def getAll(self):
+  return self.cur.fetchall()
+
+ def getNext(self):
+  return self.cur.fetchone()
+
+ def isInitialized(self):
+  try:
+   self.executeQuery("SELECT * FROM messages")
+   return True
+  except:
+   return False
+
+ def getLatestMessage(db):
+  db.executeQuery("SELECT max(id) FROM messages")
+  try:
+   return int(db.getNext()[0])
+  except:
+   return 0
+
+def dbHelper():
+ if setuptools.dbtype() == SQLITE:
+  return dbObject(dbtype=SQLITE, path=setuptools.dbpath())
+ elif setuptools.dbtype() == MYSQL:
+  return dbObject(dbtype=MYSQL, host=setuptools.dbhost(), user=setuptools.dbuser(), pwd=setuptools.dbpass(), db=setuptools.dbname())
+ else:
+  raise setuptools.SetupException()

messages.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+
+from BeautifulSoup import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.common.keys import Keys
+from selenium.common.exceptions import NoSuchElementException
+import urllib.request, urllib.error, urllib.parse, time, os
+import setuptools
+
+def status(driver):
+ if "/main/login.php" not in driver.page_source:
+  return True
+ else:
+  return False
+
+def loadPage(url,period=5,init=False, driver=driver):
+ if not (init or status(driver)):
+  login()
+ driver.get(url)
+ time.sleep(period)
+
+def loginHandler(user = setuptools.user, password = setuptools.password, driver = driver):
+ loadPage("http://www.planetromeo.com/",10,True,driver)
+  
+ curfield = driver.find_element_by_name("username")
+ curfield.send_keys(user)
+  
+ curfield = driver.find_element_by_name("passwort")
+ curfield.send_keys(password)
+  
+ curfield.send_keys(Keys.RETURN)
+ time.sleep(10)
+
+ return status()
+
+class LoginError(Exception):
+ pass
+
+def login():
+ if not (status() or loginHandler()):
+  raise LoginError("Login failed.")
+ return True
+
+def messageID(url):
+ return url.split("?")[1]
+
+def messageHandler(mid, driver):
+ loadPage("https://www.planetromeo.com/msg/?id=" + mid, driver=driver)
+
+ try:
+  links = driver.find_elements_by_partial_link_text('pix/popup.php/')
+  for link in links:
+   phototools.processURL(link.get_attribute('href'),vid)
+  except NoSuchElementException as e:
+   pass
+
+def pageHandler(driver):
+ webpage = driver.page_source
+ links = BeautifulSoup(webpage).findAll('a')
+ count = 0
+
+ for l in links:
+  url = l['href']
+  if "/msg/?id=" in url:
+   count += 1
+   mid = messageID(url)
+   if mid <= dbtools.getLatestMessage()
+    return False
+   messageHandler(mid, driver)
+
+ if count == 0:
+  return False
+
+ return True
+ 
+def siteHandler(p = 0, driver = driver):
+ loadPage("https://www.planetromeo.com/mitglieder/messages/uebersicht.php?seite=" + str(p), driver=driver)
+ if pageHandler(driver):
+  siteHandler(p+1, driver)
+
+if __name__ == "__main__":
+ driver = webdriver.Firefox()
+ if login(driver):
+  siteHandler(driver=driver)
+
+ print("KTHXBAI")
+
+ driver.close()

setup.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+import configparser, os.path, tweepy, dbtools, getpass
+
+if os.path.isfile("config.cfg"):
+ print("config.cfg already exists. Please remove it before running this script.")
+ exit(1)
+
+config = configparser.RawConfigParser()
+
+config.add_section('Database')
+
+print('''Romeotools will use a database for certain tasks. You can use a file or a MySQL database for this purpose. 
+
+If you wish to use a MySQL database, you will need the credentials for it. If you don't know what any of that means, stick with the default value and just press Enter.
+''')
+
+dbtype = input("Database type: %i (file), %i (MySQL) [%i]: " % (dbtools.SQLITE, dbtools.MYSQL, dbtools.SQLITE))
+print()
+
+try:
+ dbtype = int(dbtype)
+except:
+ pass
+
+if dbtype == dbtools.MYSQL:
+ dbhost = input("MySQL host [localhost]: ") or "localhost"
+ dbuser = input("MySQL username [twitools]: ") or "twitools"
+ dbpass = getpass.getpass("MySQL password (not echoed!): ")
+ dbname = input("MySQL database name [twitools]: ") or "twitools"
+ print()
+
+ config.set('Database', 'type', dbtype)
+ config.set('Database', 'host', dbhost)
+ config.set('Database', 'user', dbuser)
+ config.set('Database', 'pass', dbpass)
+ config.set('Database', 'name', dbname)
+
+else:
+ dbtype = dbtools.SQLITE
+ dbpath = input("Name of the database file [Database.db]: ") or "Database.db"
+ print()
+
+ config.set('Database', 'type', dbtype)
+ config.set('Database', 'path', dbpath)
+
+if dbtype == dbtools.MYSQL:
+ db = dbtools.dbObject(dbtype=dbtype, host=dbhost, user=dbuser, pwd=dbpass, db=dbname)
+else:
+ db = dbtools.dbObject(dbtype=dbtype, path=dbpath)
+if not db.isInitialized():
+ db.executeQuery("CREATE TABLE messages(`id` INTEGER NOT NULL, `text` TEXT, `sender_id` VARCHAR(60), `recipient_id` VARCHAR(60), `created_at` TEXT, PRIMARY KEY(id));")
+ db.commit()
+
+db.closeConnection()
+
+config.add_section("Romeo")
+
+print('''In the next step, we'll get you connected to PlanetRomeo. For this, we 
+will need your username and password. Please note that these will be stored on
+your hard disk in plain text. Sadly PlanetRomeo doesn't offer a better way for
+third party applications to authenticate...
+''')
+
+unam = input("Username: ")
+pwrd = getpass.getpass("Password (not echoed back!): ")
+print()
+
+config.set("Romeo", "user", unam)
+config.set("Romeo", "pass", pwrd)
+
+print("Seems like everything worked out fine. Let's write that config file...")
+
+with open('config.cfg', 'wt') as cfg:
+ config.write(cfg)
+
+print("We're all done. You can now use Twitools. Have fun!")

setuptools/__init__.py

@@ -0,0 +1,121 @@
+import configparser, csv, datetime, html.parser, itertools, os, sqlite3, sys
+
+class SetupException(Exception):
+ def __str__(self):
+  return "Seems like config.cfg has not been created yet or contains serious errors. Run setup.py to create it."
+
+def getSetting(section, setting, path = "config.cfg"):
+ config = configparser.RawConfigParser()
+ config.read(path)
+ return config.get(section, setting)
+
+def dbtype():
+ try:
+  return int(getSetting("Database", "type"))
+ except:
+  return 0 # for SQLite3
+
+### Must only be called AFTER dbtype()! ###
+
+def dbhost():
+ try:
+  return getSetting("Database", "host")
+ except:
+  raise SetupException()
+
+def dbuser():
+ try:
+  return getSetting("Database", "user")
+ except:
+  raise SetupException()
+
+def dbpass():
+ try:
+  return getSetting("Database", "pass")
+ except:
+  raise SetupException()
+
+def dbname():
+ try:
+  return getSetting("Database", "name")
+ except:
+  raise SetupException()
+
+def dbpath():
+ try:
+  return getSetting("Database", "path")
+ except:
+  return SetupException()
+
+###
+
+def user():
+ try:
+  return getSetting("Romeo", "user")
+ except:
+  raise SetupException()
+
+def password():
+ try:
+  return getSetting("Romeo", "pass")
+ except:
+  raise SetupException()
+
+def dbCheck(db, create = False):
+ if (not create and dbInitialized(db)) or (create and not dbInitialized(db)):
+  return True
+ if create:
+  raise ValueError("Provided database file " + db.path + " is already initialized. Remove it manually before trying to recreate it.")
+ raise ValueError("Provided database file " + db.path + " is not initialized. Create it using makedb.py or csvdb.py")
+
+
+def dbHelper(path, create = False):
+ db = dbObject(path)
+ dbCheck(db, create)
+ return db
+
+def dbInitialized(db):
+ return db.isInitialized()
+
+def fileExists(path):
+ return os.path.isfile(path)
+
+def getDate(date):
+ try:
+  return datetime.datetime.strptime(date, '%Y-%m-%d')
+ except ValueError:
+  raise ValueError("Dates must be in YYYY-MM-DD format.")
+
+def paginate(iterable, page_size):
+ while True:
+  i1, i2 = itertools.tee(iterable)
+  iterable, page = (itertools.islice(i1, page_size, None), list(itertools.islice(i2, page_size)))
+  if len(page) == 0:
+   break
+  yield page
+
+def parseArgs(argv):
+ args = []
+ path = None
+ nextpath = False
+
+ for a in argv[1:]:
+  if nextpath:
+   path = a
+   nextpath = False
+  elif a == "-f":
+   if path != None:
+    raise ValueError("You can only pass one database file.")
+   nextpath = True
+  else:
+   args += [a]
+
+ return args, path
+
+def printCSV(inlist):
+ writer = csv.writer(sys.stdout)
+ writer.writerows(inlist)
+
+def unescapeText(text):
+ return html.parser.HTMLParser().unescape(text).replace("'","''")
+