Commit e21fcc1a authored by Klaus-Uwe Mitterer's avatar Klaus-Uwe Mitterer

Check in stuff primarily copied over from twitools

parent a2869d70
import setuptools
import sqlite3, pymysql, pymysql.cursors
SQLITE = 0
MYSQL = 1
MARIADB = MYSQL
MIN = 0
MAX = 1
class dbObject:
# --------------------------------------------- Initialization -------------------------------------------------
def initMySQL(self, host, port, user, pwd, db):
self.conn = pymysql.connect(host = host, port = port, user = user, password = pwd, db = db, charset = "utf8mb4", cursorclass = pymysql.cursors.DictCursor)
self.cur = self.conn.cursor()
self.dbtype = MYSQL
self.host = host
self.port = port
self.user = user
self.pwd = pwd
self.db = db
def initSQLite(self, path):
self.conn = sqlite3.connect(path)
self.cur = self.conn.cursor()
self.dbtype = SQLITE
self.path = path
def __init__(self, dbtype = SQLITE, path = None, host = None, port = None, user = None, pwd = None, db = None):
if dbtype == SQLITE:
self.initSQLite(path or 'Database.db')
elif dbtype == MYSQL:
self.initMySQL(host or 'localhost', port or 3306, user, pwd, db)
else:
raise ValueError("Unknown database type %s." % str(dbtype))
# ---------------------------------------------- No more initialization ----------------------------------------
def closeConnection(self):
return self.conn.close()
def commit(self):
return self.conn.commit()
def executeQuery(self, query):
return self.cur.execute(query)
def getAll(self):
return self.cur.fetchall()
def getNext(self):
return self.cur.fetchone()
def isInitialized(self):
try:
self.executeQuery("SELECT * FROM messages")
return True
except:
return False
def getLatestMessage(db):
db.executeQuery("SELECT max(id) FROM messages")
try:
return int(db.getNext()[0])
except:
return 0
def dbHelper():
if setuptools.dbtype() == SQLITE:
return dbObject(dbtype=SQLITE, path=setuptools.dbpath())
elif setuptools.dbtype() == MYSQL:
return dbObject(dbtype=MYSQL, host=setuptools.dbhost(), user=setuptools.dbuser(), pwd=setuptools.dbpass(), db=setuptools.dbname())
else:
raise setuptools.SetupException()
#!/usr/bin/env python3
from BeautifulSoup import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
import urllib.request, urllib.error, urllib.parse, time, os
import setuptools
def status(driver):
if "/main/login.php" not in driver.page_source:
return True
else:
return False
def loadPage(url,period=5,init=False, driver=driver):
if not (init or status(driver)):
login()
driver.get(url)
time.sleep(period)
def loginHandler(user = setuptools.user, password = setuptools.password, driver = driver):
loadPage("http://www.planetromeo.com/",10,True,driver)
curfield = driver.find_element_by_name("username")
curfield.send_keys(user)
curfield = driver.find_element_by_name("passwort")
curfield.send_keys(password)
curfield.send_keys(Keys.RETURN)
time.sleep(10)
return status()
class LoginError(Exception):
pass
def login():
if not (status() or loginHandler()):
raise LoginError("Login failed.")
return True
def messageID(url):
return url.split("?")[1]
def messageHandler(mid, driver):
loadPage("https://www.planetromeo.com/msg/?id=" + mid, driver=driver)
try:
links = driver.find_elements_by_partial_link_text('pix/popup.php/')
for link in links:
phototools.processURL(link.get_attribute('href'),vid)
except NoSuchElementException as e:
pass
def pageHandler(driver):
webpage = driver.page_source
links = BeautifulSoup(webpage).findAll('a')
count = 0
for l in links:
url = l['href']
if "/msg/?id=" in url:
count += 1
mid = messageID(url)
if mid <= dbtools.getLatestMessage()
return False
messageHandler(mid, driver)
if count == 0:
return False
return True
def siteHandler(p = 0, driver = driver):
loadPage("https://www.planetromeo.com/mitglieder/messages/uebersicht.php?seite=" + str(p), driver=driver)
if pageHandler(driver):
siteHandler(p+1, driver)
if __name__ == "__main__":
driver = webdriver.Firefox()
if login(driver):
siteHandler(driver=driver)
print("KTHXBAI")
driver.close()
#!/usr/bin/env python3
import configparser, os.path, tweepy, dbtools, getpass
if os.path.isfile("config.cfg"):
print("config.cfg already exists. Please remove it before running this script.")
exit(1)
config = configparser.RawConfigParser()
config.add_section('Database')
print('''Romeotools will use a database for certain tasks. You can use a file or a MySQL database for this purpose.
If you wish to use a MySQL database, you will need the credentials for it. If you don't know what any of that means, stick with the default value and just press Enter.
''')
dbtype = input("Database type: %i (file), %i (MySQL) [%i]: " % (dbtools.SQLITE, dbtools.MYSQL, dbtools.SQLITE))
print()
try:
dbtype = int(dbtype)
except:
pass
if dbtype == dbtools.MYSQL:
dbhost = input("MySQL host [localhost]: ") or "localhost"
dbuser = input("MySQL username [twitools]: ") or "twitools"
dbpass = getpass.getpass("MySQL password (not echoed!): ")
dbname = input("MySQL database name [twitools]: ") or "twitools"
print()
config.set('Database', 'type', dbtype)
config.set('Database', 'host', dbhost)
config.set('Database', 'user', dbuser)
config.set('Database', 'pass', dbpass)
config.set('Database', 'name', dbname)
else:
dbtype = dbtools.SQLITE
dbpath = input("Name of the database file [Database.db]: ") or "Database.db"
print()
config.set('Database', 'type', dbtype)
config.set('Database', 'path', dbpath)
if dbtype == dbtools.MYSQL:
db = dbtools.dbObject(dbtype=dbtype, host=dbhost, user=dbuser, pwd=dbpass, db=dbname)
else:
db = dbtools.dbObject(dbtype=dbtype, path=dbpath)
if not db.isInitialized():
db.executeQuery("CREATE TABLE messages(`id` INTEGER NOT NULL, `text` TEXT, `sender_id` VARCHAR(60), `recipient_id` VARCHAR(60), `created_at` TEXT, PRIMARY KEY(id));")
db.commit()
db.closeConnection()
config.add_section("Romeo")
print('''In the next step, we'll get you connected to PlanetRomeo. For this, we
will need your username and password. Please note that these will be stored on
your hard disk in plain text. Sadly PlanetRomeo doesn't offer a better way for
third party applications to authenticate...
''')
unam = input("Username: ")
pwrd = getpass.getpass("Password (not echoed back!): ")
print()
config.set("Romeo", "user", unam)
config.set("Romeo", "pass", pwrd)
print("Seems like everything worked out fine. Let's write that config file...")
with open('config.cfg', 'wt') as cfg:
config.write(cfg)
print("We're all done. You can now use Twitools. Have fun!")
import configparser, csv, datetime, html.parser, itertools, os, sqlite3, sys
class SetupException(Exception):
def __str__(self):
return "Seems like config.cfg has not been created yet or contains serious errors. Run setup.py to create it."
def getSetting(section, setting, path = "config.cfg"):
config = configparser.RawConfigParser()
config.read(path)
return config.get(section, setting)
def dbtype():
try:
return int(getSetting("Database", "type"))
except:
return 0 # for SQLite3
### Must only be called AFTER dbtype()! ###
def dbhost():
try:
return getSetting("Database", "host")
except:
raise SetupException()
def dbuser():
try:
return getSetting("Database", "user")
except:
raise SetupException()
def dbpass():
try:
return getSetting("Database", "pass")
except:
raise SetupException()
def dbname():
try:
return getSetting("Database", "name")
except:
raise SetupException()
def dbpath():
try:
return getSetting("Database", "path")
except:
return SetupException()
###
def user():
try:
return getSetting("Romeo", "user")
except:
raise SetupException()
def password():
try:
return getSetting("Romeo", "pass")
except:
raise SetupException()
def dbCheck(db, create = False):
if (not create and dbInitialized(db)) or (create and not dbInitialized(db)):
return True
if create:
raise ValueError("Provided database file " + db.path + " is already initialized. Remove it manually before trying to recreate it.")
raise ValueError("Provided database file " + db.path + " is not initialized. Create it using makedb.py or csvdb.py")
def dbHelper(path, create = False):
db = dbObject(path)
dbCheck(db, create)
return db
def dbInitialized(db):
return db.isInitialized()
def fileExists(path):
return os.path.isfile(path)
def getDate(date):
try:
return datetime.datetime.strptime(date, '%Y-%m-%d')
except ValueError:
raise ValueError("Dates must be in YYYY-MM-DD format.")
def paginate(iterable, page_size):
while True:
i1, i2 = itertools.tee(iterable)
iterable, page = (itertools.islice(i1, page_size, None), list(itertools.islice(i2, page_size)))
if len(page) == 0:
break
yield page
def parseArgs(argv):
args = []
path = None
nextpath = False
for a in argv[1:]:
if nextpath:
path = a
nextpath = False
elif a == "-f":
if path != None:
raise ValueError("You can only pass one database file.")
nextpath = True
else:
args += [a]
return args, path
def printCSV(inlist):
writer = csv.writer(sys.stdout)
writer.writerows(inlist)
def unescapeText(text):
return html.parser.HTMLParser().unescape(text).replace("'","''")
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment