From c9ed677f0a311ae632072a3bcaa21aeeb6415ef1 Mon Sep 17 00:00:00 2001 From: Kumi Date: Thu, 3 Feb 2022 19:04:41 +0000 Subject: [PATCH] Initial commit - not the cleanest of projects, but it works --- .gitignore | 6 ++ __main__.py | 0 certreport.py | 52 ++++++++++++ classes/__init__.py | 0 classes/config.py | 38 +++++++++ classes/connection.py | 50 ++++++++++++ classes/database.py | 68 ++++++++++++++++ classes/logger.py | 21 +++++ classes/vessel.py | 180 ++++++++++++++++++++++++++++++++++++++++++ const.py | 13 +++ settings.dist.ini | 16 ++++ 11 files changed, 444 insertions(+) create mode 100644 .gitignore create mode 100644 __main__.py create mode 100644 certreport.py create mode 100644 classes/__init__.py create mode 100644 classes/config.py create mode 100644 classes/connection.py create mode 100644 classes/database.py create mode 100644 classes/logger.py create mode 100644 classes/vessel.py create mode 100644 const.py create mode 100644 settings.dist.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2c7543d --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.pyc +*.swp +settings.ini +venv/ +.vscode diff --git a/__main__.py b/__main__.py new file mode 100644 index 0000000..e69de29 diff --git a/certreport.py b/certreport.py new file mode 100644 index 0000000..413ad51 --- /dev/null +++ b/certreport.py @@ -0,0 +1,52 @@ +from classes.config import MonsterConfig + +from datetime import datetime, timedelta + +import argparse +import csv + + +config = MonsterConfig("settings.ini") + +certs = [] + +before = datetime.utcnow().replace(hour=0,minute=0,second=0,microsecond=0) +after = before - timedelta(days=1) + +for vessel in config.vessels: + users = dict() + + ocourses = vessel.getCourses() + courses = dict() + + for ocourse in ocourses: + courses[ocourse["id"]] = ocourse + + for ocert in vessel.getCerts(after=after.timestamp(), before=before.timestamp()): + if ocert["cert"]: + cert = dict() + + user_id = ocert["userid"] + if not (user := users.get(user_id)): + user = vessel.getUsers(id=user_id)[user_id] + users[user_id] = user + + cert["user_name"] = f'{user["firstname"]} {user["lastname"]}' + cert["user_email"] = user["email"] + cert["user_pin"] = user["custom_fields"].get("pin") + cert["course_id"] = ocert["cert"]["course"] + cert["course_shortname"] = courses[ocert["cert"]["course"]]["shortname"] + cert["course_fullname"] = courses[ocert["cert"]["course"]]["fullname"] + cert["code"] = ocert["code"] + cert["time_created"] = datetime.utcfromtimestamp(ocert["timecreated"]).strftime('%Y-%m-%d %H:%M:%S') + cert["vessel"] = vessel.name + certs.append(cert) + +certs = sorted(certs, key=lambda d: d["time_created"]) + +keys = ["user_name", "user_email", "user_pin", "course_id", "course_shortname", "course_fullname", "code", "time_created", "vessel"] + +with open('test.csv', 'a') as output_file: + dict_writer = csv.DictWriter(output_file, restval="", fieldnames=keys, delimiter=';') + dict_writer.writeheader() + dict_writer.writerows(certs) diff --git a/classes/__init__.py b/classes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/classes/config.py b/classes/config.py new file mode 100644 index 0000000..1c98088 --- /dev/null +++ b/classes/config.py @@ -0,0 +1,38 @@ +import configparser + +from pathlib import Path +from typing import Union + +from classes.vessel import Vessel + + +class MonsterConfig: + def readFile(self, path: Union[str, Path]) -> None: + """Read .ini file into MonsterConfig object + + Args: + path (str, pathlib.Path): Location of the .ini file to read + (absolute or relative to the working directory) + + Raises: + ValueError: Raised if the passed file is not a ContentMonster .ini + IOError: Raised if the file cannot be read from the provided path + """ + parser = configparser.ConfigParser() + parser.read(str(path)) + + if not "MONSTER" in parser.sections(): + raise ValueError("Config file does not contain a MONSTER section!") + + for section in parser.sections(): + # Read Vessels from the config file + if section.startswith("Vessel"): + self.vessels.append(Vessel.fromConfig(parser[section])) + + def __init__(self, path: Union[str, Path]) -> None: + """Initialize a new (empty) MonsterConfig object + """ + self.vessels = [] + + if path: + self.readFile(path) diff --git a/classes/connection.py b/classes/connection.py new file mode 100644 index 0000000..ee85751 --- /dev/null +++ b/classes/connection.py @@ -0,0 +1,50 @@ +import paramiko as pikuniku # :P + +from paramiko.client import SSHClient, WarningPolicy +from sshtunnel import SSHTunnelForwarder + +from typing import Union, Optional +from contextlib import closing + +import socket + + +class Connection: + """Class representing an SSH/SFTP connection to a Vessel + """ + + def __init__(self, vessel): + """Initialize a new Connection to a Vessel + + Args: + vessel (classes.vessel.Vessel): Vessel object to open connection to + """ + self._vessel = vessel + self._client = SSHClient() + self._client.load_system_host_keys() + self._client.set_missing_host_key_policy(WarningPolicy) + self._client.connect(vessel.host, 22, vessel.ssh_username, + vessel.ssh_password, timeout=vessel.ssh_timeout, + passphrase=vessel.ssh_passphrase) + self._transport = self._client.get_transport() + self._transport.set_keepalive(10) + self._sftp = self._client.open_sftp() + self._process = None + + def forward_tcp(self, remote=3306): + self._process = SSHTunnelForwarder( + (self._vessel.host, 22), + ssh_username=self._vessel.ssh_username, + ssh_private_key_password=self._vessel.ssh_passphrase, + remote_bind_address=("127.0.0.1", remote)) + + self._process.start() + return self._process.local_bind_port + + def __del__(self): + """Close SSH connection when ending Connection + """ + self._client.close() + + if self._process: + self._process.close() diff --git a/classes/database.py b/classes/database.py new file mode 100644 index 0000000..001e337 --- /dev/null +++ b/classes/database.py @@ -0,0 +1,68 @@ +import MySQLdb +import MySQLdb.cursors + +from typing import Union, Optional + +from classes.connection import Connection + + +class Database: + """Class wrapping MySQL database connection + """ + + def __init__(self, vessel): + """Initialize a new Database object + """ + self.vessel = vessel + self._con = None + self._ssh = None + + self._connect() + + def _execute(self, query: str, parameters: Optional[tuple] = None, ctype: Optional[MySQLdb.cursors.BaseCursor] = None) -> None: + """Execute a query on the database + + Args: + query (str): SQL query to execute + parameters (tuple, optional): Parameters to use to replace + placeholders in the query, if any. Defaults to None. + """ + cur = self.getCursor(ctype) + cur.execute(query, parameters) + self.commit() # Instantly commit after every (potential) write action + return cur.fetchall() + + def _connect(self): + if self.vessel.ssh: + self._ssh = Connection(self.vessel) + port = self._ssh.forward_tcp(3306) + host = "127.0.0.1" + else: + port = 3306 + host = self.vessel.host + + self._con = MySQLdb.connect(host=host, user=self.vessel.username, + passwd=self.vessel.password, db=self.vessel.database, port=port) + + def commit(self) -> None: + """Commit the current database transaction + + N.B.: Commit instantly after every write action to make the database + "thread-safe". Connections will time out if the database is locked for + more than five seconds. + """ + self._con.commit() + + def getCursor(self, ctype: Optional[MySQLdb.cursors.BaseCursor] = None) -> MySQLdb.cursors.BaseCursor: + """Return a cursor to operate on the MySQL database + + Returns: + MySQLdb.Cursor: Cursor object to execute queries on + """ + return self._con.cursor(ctype) + + def __del__(self): + """Close database connection on removal of the Database object + """ + self._con.close() + diff --git a/classes/logger.py b/classes/logger.py new file mode 100644 index 0000000..f452d32 --- /dev/null +++ b/classes/logger.py @@ -0,0 +1,21 @@ +import logging +import threading + +from datetime import datetime + + +class Logger: + @staticmethod + def _format(message: str, severity: str) -> str: + thread = threading.current_thread().name + datestr = str(datetime.now()) + return f"{datestr} - {thread} - {severity} - {message}" + + def debug(self, message: str) -> None: + print(self.__class__()._format(message, "DEBUG")) + + def info(self, message: str) -> None: + print(self.__class__()._format(message, "INFO")) + + def error(self, message: str) -> None: + print(self.__class__()._format(message, "ERROR")) \ No newline at end of file diff --git a/classes/vessel.py b/classes/vessel.py new file mode 100644 index 0000000..c022029 --- /dev/null +++ b/classes/vessel.py @@ -0,0 +1,180 @@ +from classes.database import Database + +from configparser import SectionProxy +from typing import Optional, Union +from datetime import datetime + +from MySQLdb.cursors import DictCursor + +from const import * + + +class Vessel: + """Class describing a Vessel + """ + @classmethod + def fromConfig(cls, config: SectionProxy): + """Create Vessel object from a Vessel section in the Config file + + Args: + config (configparser.SectionProxy): Vessel section defining a + Vessel + + Raises: + ValueError: Raised if section does not contain Address parameter + + Returns: + classes.vessel.Vessel: Vessel object for the vessel specified in + the config section + """ + + host = None + username = None + password = None + database = None + ssh = False + ssh_username = None + ssh_password = None + ssh_timeout = 100 + ssh_passphrase = None + + if "Username" in config.keys(): + username = config["Username"] + + if "Password" in config.keys(): + password = config["Password"] + + if "Database" in config.keys(): + database = config["Database"] + + if "SSH" in config.keys(): + if int(config["SSH"]) == 1: + ssh = True + + return cls(config.name.split()[1], config["Host"], username, password, database, ssh, ssh_username, ssh_password, ssh_timeout, ssh_passphrase) + + def __init__(self, name: str, host: str, username: Optional[str] = None, + password: Optional[str] = None, database: Optional[str] = None, + ssh = False, ssh_username = None, ssh_password = None, ssh_timeout = None, ssh_passphrase = None) -> None: + """Initialize new Vessel object + + Args: + name (str): Name of the Vessel + """ + self.name = name + self.host = host + self.username = username + self.password = password + self.database = database + self.ssh = ssh + self.ssh_username = ssh_username + self.ssh_password = ssh_password + self.ssh_timeout = ssh_timeout + self.ssh_passphrase = ssh_passphrase + + self.db = self.connect() + + def connect(self): + return Database(self) + + def getCourses(self) -> list: + results = self.db._execute(QUERY_COURSE, ctype=DictCursor) + return results + + def getUserInfoFields(self) -> list: + results = self.db._execute(QUERY_USER_INFO_FIELD, ctype=DictCursor) + return results + + def getUserInfoData(self, field: Optional[int] = None, user: Optional[int] = None) -> list: + query = QUERY_USER_INFO_DATA + parameters = [] + + if field: + query += " WHERE fieldid = %s" + parameters.append(int(field)) + + if user: + if query != QUERY_USER_INFO_DATA: + query += " AND " + else: + query += " WHERE " + + query += "userid = %s" + parameters.append(int(user)) + + results = self.db._execute(query, tuple(parameters), ctype=DictCursor) + + return results + + def getUsers(self, username: Optional[str] = None, id: Optional[int] = None) -> dict: + query = QUERY_USER + parameters = tuple() + + if username: + query += f" WHERE username = %s" + parameters = (username,) + + elif id: + query += f" WHERE id = %s" + parameters = (int(id),) + + results = self.db._execute(query, parameters, ctype=DictCursor) + + users = dict() + + for result in results: + user = result + result["custom_fields"] = dict() + users[user["id"]] = user + + ofields = self.getUserInfoFields() + + for ofield in ofields: + odata = self.getUserInfoData(ofield["id"], id) + + for value in odata: + users[value["userid"]]["custom_fields"][ofield["shortname"]] = value["data"] + + return users + + def getHTMLCerts(self, after: int = 0, before: int = int(datetime.now().timestamp())): + results = self.db._execute(f"{QUERY_HTML_CERT_ISSUES} {QUERY_WHERE_TIMESTAMPS % {'column': 'timecreated', 'after': after, 'before': before}}", ctype=DictCursor) + ocerts = self.db._execute(QUERY_HTML_CERT, ctype=DictCursor) + + certs = dict() + + for ocert in ocerts: + certs[ocert["id"]] = ocert + + for result in results: + try: + result["cert"] = certs[result["htmlcertid"]] + except KeyError: + result["cert"] = None + + result["certtype"] = "htmlcert" + + return results + + def getCustomCerts(self, after: int = 0, before: int = int(datetime.now().timestamp())): + results = self.db._execute(f"{QUERY_CUSTOM_CERT_ISSUES} {QUERY_WHERE_TIMESTAMPS % {'column': 'timecreated', 'after': after, 'before': before}}", ctype=DictCursor) + ocerts = self.db._execute(QUERY_CUSTOM_CERT, ctype=DictCursor) + + certs = dict() + + for ocert in ocerts: + certs[ocert["id"]] = ocert + + for result in results: + try: + result["cert"] = certs[result["customcertid"]] + except KeyError: + result["cert"] = None + + result["certtype"] = "customcert" + + return results + + def getCerts(self, after: int = 0, before: int = int(datetime.now().timestamp())): + return sorted(self.getHTMLCerts(after, before) + self.getCustomCerts(after, before), key=lambda d: d["timecreated"]) + diff --git a/const.py b/const.py new file mode 100644 index 0000000..9e31292 --- /dev/null +++ b/const.py @@ -0,0 +1,13 @@ +QUERY_HTML_CERT_ISSUES = "SELECT * FROM mdl_htmlcert_issues" +QUERY_HTML_CERT = "SELECT * FROM mdl_htmlcert" + +QUERY_CUSTOM_CERT_ISSUES = "SELECT * FROM mdl_customcert_issues" +QUERY_CUSTOM_CERT = "SELECT * FROM mdl_customcert" + +QUERY_USER = "SELECT * FROM mdl_user" +QUERY_USER_INFO_FIELD = "SELECT * FROM mdl_user_info_field" +QUERY_USER_INFO_DATA = "SELECT * FROM mdl_user_info_data" + +QUERY_COURSE = "SELECT * FROM mdl_course" + +QUERY_WHERE_TIMESTAMPS = "WHERE %(column)s >= %(after)i AND %(column)s < %(before)i" diff --git a/settings.dist.ini b/settings.dist.ini new file mode 100644 index 0000000..1daface --- /dev/null +++ b/settings.dist.ini @@ -0,0 +1,16 @@ +[MONSTER] + +[Vessel vessel1] +Host = 10.12.13.14 +Username = monster +Password = jdsfskafjaf +Database = moodle +ssh = 0 + +[Vessel vessel2] +Host = 192.168.95192.1 +Username = monster +Password = jfaskldfjklasfh +Database = moodle +ssh = 1 +