Forked romeotools for ScriptzBaseTools
This commit is contained in:
Klaus-Uwe Mitterer
parent
ff759fbf7b
commit
aa2795f46d
115
handler.py
115
handler.py
|
|
@ -4,11 +4,11 @@ from bs4 import BeautifulSoup
|
|||
from selenium import webdriver
|
||||
from selenium.webdriver.common.keys import Keys
|
||||
from selenium.common.exceptions import NoSuchElementException
|
||||
import multiprocessing, urllib.request, urllib.error, urllib.parse, time, os
|
||||
import dbtools, phototools, setuptools
|
||||
import glob, multiprocessing, re, requests, urllib.request, urllib.error, urllib.parse, time, os
|
||||
import setuptools
|
||||
|
||||
def status(driver):
|
||||
if "Unauthorised Access" not in driver.page_source and "/main/login.php" not in driver.page_source and len(driver.page_source) > 100:
|
||||
if "Benutzername oder E-Mail-Adresse:" not in driver.page_source and 'href="login/"' not in driver.page_source:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
|
@ -20,13 +20,12 @@ def loadPage(url, driver, period=5,init=False):
|
|||
time.sleep(period)
|
||||
|
||||
def loginHandler(driver, user = setuptools.user(), password = setuptools.password()):
|
||||
loadPage("https://classic.planetromeo.com/",driver,3,True)
|
||||
loadPage("https://classic.planetromeo.com/main/login.php",driver,3,True)
|
||||
loadPage("https://scriptzbase.org/login/",driver,3,True)
|
||||
|
||||
curfield = driver.find_element_by_name("username")
|
||||
curfield = driver.find_element_by_name("login")
|
||||
curfield.send_keys(user)
|
||||
|
||||
curfield = driver.find_element_by_name("passwort")
|
||||
curfield = driver.find_element_by_name("password")
|
||||
curfield.send_keys(password)
|
||||
|
||||
curfield.send_keys(Keys.RETURN)
|
||||
|
|
@ -42,99 +41,39 @@ def login(driver):
|
|||
raise LoginError("Login failed.")
|
||||
return True
|
||||
|
||||
def messageID(url):
|
||||
return url.split("=")[-1]
|
||||
|
||||
def quickshareHandler(driver, url, sender):
|
||||
nurl = "https://classic.planetromeo.com/" + url if "planetromeo.com" not in url else url
|
||||
loadPage(nurl)
|
||||
def pageHandler(driver):
|
||||
juha = BeautifulSoup(driver.page_source, "html5lib")
|
||||
|
||||
try:
|
||||
links = juha.findAll("a")
|
||||
for link in links:
|
||||
try:
|
||||
purl = "https://classic.planetromeo.com/" + link["data-pic"] if "planetromeo.com" not in link["data-pic"] else link["data-pic"]
|
||||
phototools.processURL(purl, sender, shutup=True)
|
||||
except:
|
||||
pass
|
||||
except:
|
||||
pass
|
||||
session = requests.Session()
|
||||
kekse = driver.get_cookies()
|
||||
|
||||
def messageHandler(sender, recipient, mid, date, driver, mode = 0, db = dbtools.dbHelper()):
|
||||
if mode == 0:
|
||||
loadPage("https://classic.planetromeo.com/msg/?id=" + mid, driver)
|
||||
else:
|
||||
loadPage("https://classic.planetromeo.com/msg/?type=sent&id=" + mid, driver)
|
||||
juha = BeautifulSoup(driver.page_source, "html5lib")
|
||||
text = juha.select("div.msg div")[0]
|
||||
for keks in kekse:
|
||||
session.cookies.set(cookie["name"], cookie["value"])
|
||||
|
||||
db.executeQuery("INSERT INTO messages(id, text, sender_id, recipient_id, created_at) VALUES('%s', '%s', '%s', '%s', '%s');" % (mid, setuptools.unescapeText(text.text).strip(), sender, recipient, date))
|
||||
db.commit()
|
||||
for a in juha.findAll("a"):
|
||||
if "/download?version=" in a["href"]:
|
||||
fid = a["href"].split("=")[1]
|
||||
if not glob.glob("files/sbd%s*" % fid):
|
||||
res = session.get(a["href"])
|
||||
fname = re.findall("filename=(.+)", res.headers["content-disposition"])
|
||||
|
||||
try:
|
||||
links = juha.findAll("a")
|
||||
for link in links:
|
||||
if "/pix/popup.php/" in link["href"]:
|
||||
phototools.processURL(link["href"], sender)
|
||||
try:
|
||||
db.executeQuery("INSERT INTO photos(mid, pid) VALUES('%s', '%s');" % (mid, phototools.parseurl(link["href"]).split('/')[-1]))
|
||||
except:
|
||||
pass
|
||||
if "/quickshare/" in link["href"]:
|
||||
quickshareHandler(driver, link["href"], sender)
|
||||
except:
|
||||
pass
|
||||
with open("files/sbd%s_%s" % (fid, fname), "wb") as out:
|
||||
out.write(res.content)
|
||||
|
||||
def pageHandler(driver, db = dbtools.dbHelper()):
|
||||
mode = 0
|
||||
abort = True
|
||||
juha = BeautifulSoup(driver.page_source, "html5lib")
|
||||
def siteHandler(driver, p = 1):
|
||||
loadPage("https://scriptzbase.org/nulled_scripts/categories/scripts-templates.145/?page=" + str(p), driver)
|
||||
|
||||
if "sent" in driver.current_url:
|
||||
mode = 1
|
||||
if driver.current_url[-len(str(p)):] == str(p):
|
||||
if pageHandler(driver):
|
||||
siteHandler(driver, p+1)
|
||||
|
||||
try:
|
||||
for msg in juha.select("table.messageCenter tr")[1:]:
|
||||
try:
|
||||
data = msg.findAll('td')
|
||||
user = data[1].string
|
||||
mid = messageID(data[2].find("a")["href"])
|
||||
date = data[3].string
|
||||
if not db.checkID(mid):
|
||||
abort = False
|
||||
if mode == 0:
|
||||
messageHandler(user, setuptools.user(), mid, date, driver, mode, db)
|
||||
else:
|
||||
messageHandler(setuptools.user(), user, mid, date, driver, mode, db)
|
||||
except IndexError:
|
||||
pass
|
||||
except IndexError:
|
||||
return False
|
||||
|
||||
if abort:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
def siteHandler(driver, mode = 0, p = 0, db = dbtools.dbHelper()):
|
||||
if mode == 0:
|
||||
loadPage("https://classic.planetromeo.com/mitglieder/messages/uebersicht.php?view=all&seite=" + str(p), driver)
|
||||
else:
|
||||
loadPage("https://classic.planetromeo.com/mitglieder/messages/uebersicht.php?view=sent&seite=" + str(p), driver)
|
||||
|
||||
if pageHandler(driver, db):
|
||||
siteHandler(driver, mode, p+1, db)
|
||||
|
||||
def mainHandler(driver, db):
|
||||
def mainHandler(driver):
|
||||
loginHandler(driver)
|
||||
siteHandler(driver, 0, db=db)
|
||||
siteHandler(driver, 1, db=db)
|
||||
siteHandler(driver)
|
||||
|
||||
if __name__ == "__main__":
|
||||
db = dbtools.dbHelper()
|
||||
caps = webdriver.DesiredCapabilities().PHANTOMJS.copy()
|
||||
caps["phantoms.page.settings.userAgent"] = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0"
|
||||
driver = webdriver.PhantomJS(desired_capabilities=caps)
|
||||
mainHandler(driver, db)
|
||||
mainHandler(driver)
|
||||
driver.close()
|
||||
|
|
|
|||
|
|
@ -1,43 +0,0 @@
|
|||
import argparse, os, requests, sys
|
||||
|
||||
def log(string, shutup = False, output = sys.stderr):
|
||||
if not shutup:
|
||||
print(string, file=output)
|
||||
|
||||
def verboselog(string, verbose = False, output = sys.stderr):
|
||||
if verbose:
|
||||
log(string, False, output)
|
||||
|
||||
def fileDownloader(url, sender = False, directory = False):
|
||||
os.makedirs("%s/%s" % (directory or ".", sender or "."), exist_ok=True)
|
||||
filename = "%s/%s/%s" % (directory or ".", sender or ".", url.split('/')[-1])
|
||||
remote = requests.get(url, stream=True)
|
||||
with open(filename, 'wb') as outfile:
|
||||
for chunk in remote.iter_content(chunk_size=1024):
|
||||
if chunk:
|
||||
outfile.write(chunk)
|
||||
outfile.flush()
|
||||
|
||||
def urlparse(url):
|
||||
if "/img/usr/" in url:
|
||||
return url
|
||||
if "/auswertung/pix/popup.php/" in url:
|
||||
return "http://www.planetromeo.com/img/usr/%s" % url.split("/")[-1].split("?")[0]
|
||||
else:
|
||||
raise ValueError("%s is not a valid URL" % url)
|
||||
|
||||
def processURL(url, sender = False, directory = "photos", geturls = False, verbose = False, shutup = False):
|
||||
verboselog("Processing URL %s..." % url, verbose)
|
||||
try:
|
||||
purl = urlparse(url)
|
||||
except ValueError as e:
|
||||
log("Notice: %s. Skipping." % e, shutup)
|
||||
else:
|
||||
if purl == url:
|
||||
log("Warning: You may have copied the image URL rather than the link URL from the message window. Use the link URL instead!", shutup)
|
||||
if geturls:
|
||||
print(purl)
|
||||
else:
|
||||
verboselog("Downloading file to directory %s..." % sender, verbose)
|
||||
fileDownloader(purl, sender, directory)
|
||||
|
||||
12
setup.py
12
setup.py
|
|
@ -10,7 +10,7 @@ config = configparser.RawConfigParser()
|
|||
|
||||
config.add_section('Database')
|
||||
|
||||
print('''Romeotools will use a database for certain tasks. You can use a file or a MySQL database for this purpose.
|
||||
print('''SBTools will use a database for certain tasks. You can use a file or a MySQL database for this purpose.
|
||||
|
||||
If you wish to use a MySQL database, you will need the credentials for it. If you don't know what any of that means, stick with the default value and just press Enter.
|
||||
''')
|
||||
|
|
@ -55,11 +55,11 @@ if not db.isInitialized():
|
|||
|
||||
db.closeConnection()
|
||||
|
||||
config.add_section("Romeo")
|
||||
config.add_section("SB")
|
||||
|
||||
print('''In the next step, we'll get you connected to PlanetRomeo. For this, we
|
||||
print('''In the next step, we'll get you connected to ScriptzBase. For this, we
|
||||
will need your username and password. Please note that these will be stored on
|
||||
your hard disk in plain text. Sadly PlanetRomeo doesn't offer a better way for
|
||||
your hard disk in plain text. Sadly ScriptzBase doesn't offer a better way for
|
||||
third party applications to authenticate...
|
||||
''')
|
||||
|
||||
|
|
@ -67,8 +67,8 @@ unam = input("Username: ")
|
|||
pwrd = getpass.getpass("Password (not echoed back!): ")
|
||||
print()
|
||||
|
||||
config.set("Romeo", "user", unam)
|
||||
config.set("Romeo", "pass", pwrd)
|
||||
config.set("SB", "user", unam)
|
||||
config.set("SB", "pass", pwrd)
|
||||
|
||||
print("Seems like everything worked out fine. Let's write that config file...")
|
||||
|
||||
|
|
|
|||
|
|
@ -51,13 +51,13 @@ def dbpath():
|
|||
|
||||
def user():
|
||||
try:
|
||||
return getSetting("Romeo", "user")
|
||||
return getSetting("SB", "user")
|
||||
except:
|
||||
raise SetupException()
|
||||
|
||||
def password():
|
||||
try:
|
||||
return getSetting("Romeo", "pass")
|
||||
return getSetting("SB", "pass")
|
||||
except:
|
||||
raise SetupException()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue