feat: Enhance stability and configurability
- Extended copyright to reflect the current year. - Incremented project version to indicate new features and fixes. - Added a new script entry for easier execution, increasing utility and accessibility. - Updated project URLs for better alignment with current infrastructure. - Refactored settings path for simplicity and consistency across deployments. - Improved code readability and maintenance across several modules by cleaning up redundant code, adding missing type annotations, and ensuring consistent code formatting. - Enhanced logging capabilities and error handling to improve diagnostics and troubleshooting, supporting more robust error recovery mechanisms. - Implemented more graceful handling of termination signals to ensure clean shutdown and resource cleanup, enhancing the robustness of the application in production environments. - Introduced command-line argument parsing for configuration file path customization, improving flexibility in different runtime environments. These changes collectively improve the project's maintainability, reliability, and user experience, laying a stronger foundation for future development.
This commit is contained in:
parent
6df9d7b9dc
commit
e82ccb2701
2
LICENSE
2
LICENSE
|
@ -1,4 +1,4 @@
|
||||||
Copyright (c) 2021-2022 Kumi Systems e.U. <office@kumi.systems>
|
Copyright (c) 2021-2024 Kumi Systems e.U. <office@kumi.systems>
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
|
|
@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "contentmonster"
|
name = "contentmonster"
|
||||||
version = "0.0.6"
|
version = "0.0.7"
|
||||||
authors = [
|
authors = [
|
||||||
{ name="Kumi Systems e.U.", email="office@kumi.systems" },
|
{ name="Kumi Systems e.U.", email="office@kumi.systems" },
|
||||||
]
|
]
|
||||||
|
@ -22,6 +22,10 @@ dependencies = [
|
||||||
"watchdog"
|
"watchdog"
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[project.scripts]
|
||||||
|
contentmonster = "contentmonster.worker:main"
|
||||||
|
|
||||||
[project.urls]
|
[project.urls]
|
||||||
"Homepage" = "https://kumig.it/kumisystems/contentmonster"
|
"Homepage" = "https://git.private.coffee/kumisystems/contentmonster"
|
||||||
"Bug Tracker" = "https://kumig.it/kumisystems/contentmonster/issues"
|
"Bug Tracker" = "https://git.private.coffee/kumisystems/contentmonster/issues"
|
||||||
|
"Source Code" = "https://git.private.coffee/kumisystems/contentmonster"
|
|
@ -15,8 +15,8 @@ class Chunk:
|
||||||
chunksize (int): Size of each chunk in bytes
|
chunksize (int): Size of each chunk in bytes
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
classes.chunk.Chunk: A Chunk object containing the portion of the
|
classes.chunk.Chunk: A Chunk object containing the portion of the
|
||||||
File object beginning at (count * chunksize) bytes and ending at
|
File object beginning at (count * chunksize) bytes and ending at
|
||||||
((count + 1) * chunksize - 1) bytes
|
((count + 1) * chunksize - 1) bytes
|
||||||
"""
|
"""
|
||||||
return fileobj.getChunk(count, chunksize)
|
return fileobj.getChunk(count, chunksize)
|
||||||
|
@ -26,7 +26,7 @@ class Chunk:
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
fileobj (classes.file.File): The file object from local storage
|
fileobj (classes.file.File): The file object from local storage
|
||||||
count (int): Position of the current chunk in the list of total
|
count (int): Position of the current chunk in the list of total
|
||||||
chunks (first index: 0) or -1 to get the complete file
|
chunks (first index: 0) or -1 to get the complete file
|
||||||
data (bytes): Content of the chunk
|
data (bytes): Content of the chunk
|
||||||
"""
|
"""
|
||||||
|
@ -49,3 +49,6 @@ class Chunk:
|
||||||
str: SHA256 hash of Chunk.data
|
str: SHA256 hash of Chunk.data
|
||||||
"""
|
"""
|
||||||
return hashlib.sha256(self.data).hexdigest()
|
return hashlib.sha256(self.data).hexdigest()
|
||||||
|
|
||||||
|
def getFileSize(self) -> int:
|
||||||
|
return len(self.data)
|
||||||
|
|
|
@ -41,17 +41,14 @@ class MonsterConfig:
|
||||||
for section in parser.sections():
|
for section in parser.sections():
|
||||||
# Read Directories from the config file
|
# Read Directories from the config file
|
||||||
if section.startswith("Directory"):
|
if section.startswith("Directory"):
|
||||||
self.directories.append(
|
self.directories.append(Directory.fromConfig(parser[section]))
|
||||||
Directory.fromConfig(parser[section]))
|
|
||||||
|
|
||||||
# Read Vessels from the config file
|
# Read Vessels from the config file
|
||||||
elif section.startswith("Vessel"):
|
elif section.startswith("Vessel"):
|
||||||
self.vessels.append(
|
self.vessels.append(Vessel.fromConfig(parser[section], dbclass))
|
||||||
Vessel.fromConfig(parser[section], dbclass))
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
"""Initialize a new (empty) MonsterConfig object
|
"""Initialize a new (empty) MonsterConfig object"""
|
||||||
"""
|
|
||||||
self.directories = []
|
self.directories = []
|
||||||
self.vessels = []
|
self.vessels = []
|
||||||
self.chunksize = 10485760 # Default: 10 MiB
|
self.chunksize = 10485760 # Default: 10 MiB
|
||||||
|
|
|
@ -9,10 +9,11 @@ from typing import Union, Optional
|
||||||
import errno
|
import errno
|
||||||
import stat
|
import stat
|
||||||
|
|
||||||
|
from .logger import Logger
|
||||||
|
|
||||||
|
|
||||||
class Connection:
|
class Connection:
|
||||||
"""Class representing an SSH/SFTP connection to a Vessel
|
"""Class representing an SSH/SFTP connection to a Vessel"""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, vessel):
|
def __init__(self, vessel):
|
||||||
"""Initialize a new Connection to a Vessel
|
"""Initialize a new Connection to a Vessel
|
||||||
|
@ -24,12 +25,20 @@ class Connection:
|
||||||
self._client = SSHClient()
|
self._client = SSHClient()
|
||||||
self._client.load_system_host_keys()
|
self._client.load_system_host_keys()
|
||||||
self._client.set_missing_host_key_policy(WarningPolicy)
|
self._client.set_missing_host_key_policy(WarningPolicy)
|
||||||
self._client.connect(vessel.address, vessel.port, vessel.username,
|
self._client.connect(
|
||||||
vessel.password, timeout=vessel.timeout,
|
vessel.address,
|
||||||
passphrase=vessel.passphrase)
|
vessel.port,
|
||||||
|
vessel.username,
|
||||||
|
vessel.password,
|
||||||
|
timeout=vessel.timeout,
|
||||||
|
passphrase=vessel.passphrase,
|
||||||
|
auth_timeout=vessel.timeout,
|
||||||
|
banner_timeout=vessel.timeout,
|
||||||
|
)
|
||||||
self._transport = self._client.get_transport()
|
self._transport = self._client.get_transport()
|
||||||
self._transport.set_keepalive(10)
|
self._transport.set_keepalive(10)
|
||||||
self._sftp = self._client.open_sftp()
|
self._sftp = self._client.open_sftp()
|
||||||
|
self._logger = Logger()
|
||||||
|
|
||||||
def _exists(self, path: Union[str, Path]) -> bool:
|
def _exists(self, path: Union[str, Path]) -> bool:
|
||||||
"""Check if a path exists on the Vessel. Symlinks are not followed.
|
"""Check if a path exists on the Vessel. Symlinks are not followed.
|
||||||
|
@ -97,20 +106,35 @@ class Connection:
|
||||||
"""
|
"""
|
||||||
return self._sftp.remove(str(path))
|
return self._sftp.remove(str(path))
|
||||||
|
|
||||||
|
def _size(self, path: Union[str, Path]) -> int:
|
||||||
|
"""Get the size of a file on the Vessel
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path (str, pathlib.Path): Path of the file to check
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
int: Size of the file in bytes
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
FileNotFoundError: Raised if no file is found at the given path
|
||||||
|
"""
|
||||||
|
return self._sftp.stat(str(path)).st_size
|
||||||
|
|
||||||
def assertTempDirectory(self) -> None:
|
def assertTempDirectory(self) -> None:
|
||||||
"""Make sure that the temp directory exists on the Vessel
|
"""Make sure that the temp directory exists on the Vessel
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
ValueError: Raised if the path is already in use on the vessel but
|
ValueError: Raised if the path is already in use on the vessel but
|
||||||
is not a directory.
|
is not a directory.
|
||||||
IOError: Raised if the directory does not exist but cannot be
|
IOError: Raised if the directory does not exist but cannot be
|
||||||
created.
|
created.
|
||||||
"""
|
"""
|
||||||
if not self._exists(self._vessel.tempdir):
|
if not self._exists(self._vessel.tempdir):
|
||||||
self._mkdir(self._vessel.tempdir)
|
self._mkdir(self._vessel.tempdir)
|
||||||
elif not self._isdir(self._vessel.tempdir):
|
elif not self._isdir(self._vessel.tempdir):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"{self._vessel.tempdir} exists but is not a directory on Vessel {self._vessel.name}!")
|
f"{self._vessel.tempdir} exists but is not a directory on Vessel {self._vessel.name}!"
|
||||||
|
)
|
||||||
|
|
||||||
def assertDirectories(self, directory) -> None:
|
def assertDirectories(self, directory) -> None:
|
||||||
"""Make sure that destination and temp directories exist on the Vessel
|
"""Make sure that destination and temp directories exist on the Vessel
|
||||||
|
@ -122,7 +146,7 @@ class Connection:
|
||||||
Raises:
|
Raises:
|
||||||
ValueError: Raised if a path is already in use on the vessel but
|
ValueError: Raised if a path is already in use on the vessel but
|
||||||
not a directory.
|
not a directory.
|
||||||
IOError: Raised if a directory that does not exist cannot be
|
IOError: Raised if a directory that does not exist cannot be
|
||||||
created.
|
created.
|
||||||
"""
|
"""
|
||||||
for d in [directory.location, self._vessel.tempdir]:
|
for d in [directory.location, self._vessel.tempdir]:
|
||||||
|
@ -130,15 +154,18 @@ class Connection:
|
||||||
self._mkdir(d)
|
self._mkdir(d)
|
||||||
elif not self._isdir(d):
|
elif not self._isdir(d):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"{d} exists but is not a directory on Vessel {self._vessel.name}!")
|
f"{d} exists but is not a directory on Vessel {self._vessel.name}!"
|
||||||
|
)
|
||||||
|
|
||||||
def assertChunkComplete(self, chunk, path: Optional[Union[str, Path]] = None) -> bool:
|
def assertChunkComplete(
|
||||||
|
self, chunk: "Chunk", path: Optional[Union[str, Path]] = None
|
||||||
|
) -> bool:
|
||||||
"""Check if a Chunk has been uploaded correctly
|
"""Check if a Chunk has been uploaded correctly
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
chunk (classes.chunk.Chunk): Chunk object to verify upload for
|
chunk (classes.chunk.Chunk): Chunk object to verify upload for
|
||||||
path (str, pathlib.Path, optional): Optional path at which to
|
path (str, pathlib.Path, optional): Optional path at which to
|
||||||
check. If None, will get default path from Chunk object.
|
check. If None, will get default path from Chunk object.
|
||||||
Defaults to None.
|
Defaults to None.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
@ -154,10 +181,29 @@ class Connection:
|
||||||
o.channel.recv_exit_status()
|
o.channel.recv_exit_status()
|
||||||
|
|
||||||
# Remove the file if it is broken
|
# Remove the file if it is broken
|
||||||
if not o.readline().split()[0] == chunk.getHash():
|
complete_output = o.readlines()
|
||||||
|
filehash = complete_output[0].split()[0]
|
||||||
|
chunkhash = chunk.getHash()
|
||||||
|
|
||||||
|
if filehash != chunkhash:
|
||||||
|
self._logger.error(
|
||||||
|
f"Hash mismatch for {chunk.file.uuid} chunk {chunk.count}: {filehash} != {chunkhash}"
|
||||||
|
)
|
||||||
|
|
||||||
|
filesize = self._size(path)
|
||||||
|
chunksize = chunk.getFileSize()
|
||||||
|
|
||||||
|
self._logger.debug(
|
||||||
|
f"File size: {filesize} bytes, expected chunk size: {chunksize} bytes"
|
||||||
|
)
|
||||||
|
self._logger.debug("\n".join(complete_output))
|
||||||
|
|
||||||
self._remove(path)
|
self._remove(path)
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
self._logger.debug(
|
||||||
|
f"Hash match for {chunk.file.uuid} chunk {chunk.count}: {filehash} == {chunkhash}"
|
||||||
|
)
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
@ -185,16 +231,30 @@ class Connection:
|
||||||
numchunks = remotefile.getStatus() + 1
|
numchunks = remotefile.getStatus() + 1
|
||||||
|
|
||||||
# Get files in correct order to concatenate
|
# Get files in correct order to concatenate
|
||||||
files = " ".join(
|
files = [
|
||||||
[str(self._vessel.tempdir / f"{remotefile.file.uuid}_{i}.part") for i in range(numchunks)])
|
str(self._vessel.tempdir / f"{remotefile.file.uuid}_{i}.part")
|
||||||
|
for i in range(numchunks)
|
||||||
|
]
|
||||||
|
|
||||||
|
for filepath in files:
|
||||||
|
if not self._exists(filepath):
|
||||||
|
self._logger.error(f"{filepath} does not exist...?")
|
||||||
|
return False
|
||||||
|
|
||||||
completefile = remotefile.file.getChunk(-1)
|
completefile = remotefile.file.getChunk(-1)
|
||||||
outname = completefile.getTempName()
|
outname = completefile.getTempName()
|
||||||
outpath = self._vessel.tempdir / outname
|
outpath = self._vessel.tempdir / outname
|
||||||
_, o, _ = self._client.exec_command(f"cat {files} > {outpath}")
|
|
||||||
|
|
||||||
# Block for command to complete
|
if self._exists(outpath):
|
||||||
o.channel.recv_exit_status()
|
self._remove(outpath)
|
||||||
|
|
||||||
|
for f in files:
|
||||||
|
_, o, _ = self._client.exec_command(f"cat {f} >> {outpath}")
|
||||||
|
|
||||||
|
# Block for command to complete
|
||||||
|
o.channel.recv_exit_status()
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
def assertComplete(self, remotefile, allow_retry: bool = False) -> bool:
|
def assertComplete(self, remotefile, allow_retry: bool = False) -> bool:
|
||||||
"""Check if File has been reassembled from Chunks correctly
|
"""Check if File has been reassembled from Chunks correctly
|
||||||
|
@ -230,7 +290,7 @@ class Connection:
|
||||||
"""Moves reassembled file to output destination
|
"""Moves reassembled file to output destination
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
remotefile (classes.remotefile.RemoteFile): RemoteFile object
|
remotefile (classes.remotefile.RemoteFile): RemoteFile object
|
||||||
describing the uploaded File.
|
describing the uploaded File.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
@ -239,7 +299,8 @@ class Connection:
|
||||||
completefile = remotefile.file.getChunk(-1)
|
completefile = remotefile.file.getChunk(-1)
|
||||||
destination = remotefile.file.getFullPath()
|
destination = remotefile.file.getFullPath()
|
||||||
self._sftp.posix_rename(
|
self._sftp.posix_rename(
|
||||||
str(self._vessel.tempdir / completefile.getTempName()), str(destination))
|
str(self._vessel.tempdir / completefile.getTempName()), str(destination)
|
||||||
|
)
|
||||||
|
|
||||||
# Make sure that file has actually been created at destination
|
# Make sure that file has actually been created at destination
|
||||||
self._sftp.stat(str(destination))
|
self._sftp.stat(str(destination))
|
||||||
|
@ -255,12 +316,10 @@ class Connection:
|
||||||
return f.split("_")[0]
|
return f.split("_")[0]
|
||||||
|
|
||||||
def clearTempDir(self) -> None:
|
def clearTempDir(self) -> None:
|
||||||
"""Clean up the temporary directory on the Vessel
|
"""Clean up the temporary directory on the Vessel"""
|
||||||
"""
|
|
||||||
for f in self._listdir(self._vessel.tempdir):
|
for f in self._listdir(self._vessel.tempdir):
|
||||||
self._remove(self._vessel.tempdir / f)
|
self._remove(self._vessel.tempdir / f)
|
||||||
|
|
||||||
def __del__(self):
|
def __del__(self):
|
||||||
"""Close SSH connection when ending Connection
|
"""Close SSH connection when ending Connection"""
|
||||||
"""
|
|
||||||
self._client.close()
|
self._client.close()
|
||||||
|
|
|
@ -4,10 +4,11 @@ import uuid
|
||||||
|
|
||||||
from typing import Union, Optional
|
from typing import Union, Optional
|
||||||
|
|
||||||
|
from .logger import Logger
|
||||||
|
|
||||||
|
|
||||||
class Database:
|
class Database:
|
||||||
"""Class wrapping sqlite3 database connection
|
"""Class wrapping sqlite3 database connection"""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, filename: Optional[Union[str, pathlib.Path]] = None):
|
def __init__(self, filename: Optional[Union[str, pathlib.Path]] = None):
|
||||||
"""Initialize a new Database object
|
"""Initialize a new Database object
|
||||||
|
@ -17,22 +18,34 @@ class Database:
|
||||||
database to use. If None, use "database.sqlite3" in project base
|
database to use. If None, use "database.sqlite3" in project base
|
||||||
directory. Defaults to None.
|
directory. Defaults to None.
|
||||||
"""
|
"""
|
||||||
filename = filename or pathlib.Path(
|
filename = (
|
||||||
__file__).parent.parent.absolute() / "database.sqlite3"
|
filename
|
||||||
|
or pathlib.Path(__file__).parent.parent.absolute() / "database.sqlite3"
|
||||||
|
)
|
||||||
self._con = sqlite3.connect(filename)
|
self._con = sqlite3.connect(filename)
|
||||||
self.migrate()
|
self.migrate()
|
||||||
|
|
||||||
def _execute(self, query: str, parameters: Optional[tuple] = None) -> None:
|
def _execute(
|
||||||
|
self, query: str, parameters: Optional[tuple] = None, retry: bool = True
|
||||||
|
) -> None:
|
||||||
"""Execute a query on the database
|
"""Execute a query on the database
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
query (str): SQL query to execute
|
query (str): SQL query to execute
|
||||||
parameters (tuple, optional): Parameters to use to replace
|
parameters (tuple, optional): Parameters to use to replace
|
||||||
placeholders in the query, if any. Defaults to None.
|
placeholders in the query, if any. Defaults to None.
|
||||||
|
retry (bool, optional): Whether to retry the query if it fails due to
|
||||||
|
a locked database. Defaults to True.
|
||||||
"""
|
"""
|
||||||
cur = self.getCursor()
|
try:
|
||||||
cur.execute(query, parameters)
|
cur = self.getCursor()
|
||||||
self.commit() # Instantly commit after every (potential) write action
|
cur.execute(query, parameters)
|
||||||
|
self.commit() # Instantly commit after every (potential) write action
|
||||||
|
except sqlite3.OperationalError as e:
|
||||||
|
self._logger.error(f"An error occurred while writing to the database: {e}")
|
||||||
|
if retry:
|
||||||
|
return self._execute(query, parameters, False)
|
||||||
|
raise
|
||||||
|
|
||||||
def commit(self) -> None:
|
def commit(self) -> None:
|
||||||
"""Commit the current database transaction
|
"""Commit the current database transaction
|
||||||
|
@ -60,7 +73,8 @@ class Database:
|
||||||
cur = self.getCursor()
|
cur = self.getCursor()
|
||||||
try:
|
try:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"SELECT value FROM contentmonster_settings WHERE key = 'dbversion'")
|
"SELECT value FROM contentmonster_settings WHERE key = 'dbversion'"
|
||||||
|
)
|
||||||
assert (version := cur.fetchone())
|
assert (version := cur.fetchone())
|
||||||
return int(version[0])
|
return int(version[0])
|
||||||
except (sqlite3.OperationalError, AssertionError):
|
except (sqlite3.OperationalError, AssertionError):
|
||||||
|
@ -78,8 +92,10 @@ class Database:
|
||||||
hash = fileobj.getHash()
|
hash = fileobj.getHash()
|
||||||
|
|
||||||
cur = self.getCursor()
|
cur = self.getCursor()
|
||||||
cur.execute("SELECT uuid, checksum FROM contentmonster_file WHERE directory = ? AND name = ?",
|
cur.execute(
|
||||||
(fileobj.directory.name, fileobj.name))
|
"SELECT uuid, checksum FROM contentmonster_file WHERE directory = ? AND name = ?",
|
||||||
|
(fileobj.directory.name, fileobj.name),
|
||||||
|
)
|
||||||
|
|
||||||
fileuuid = None
|
fileuuid = None
|
||||||
|
|
||||||
|
@ -102,7 +118,7 @@ class Database:
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
fileobj (classes.file.File): File object to add to database
|
fileobj (classes.file.File): File object to add to database
|
||||||
hash (str, optional): Checksum of the file, if already known.
|
hash (str, optional): Checksum of the file, if already known.
|
||||||
Defaults to None.
|
Defaults to None.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
@ -110,8 +126,10 @@ class Database:
|
||||||
"""
|
"""
|
||||||
hash = hash or fileobj.getHash()
|
hash = hash or fileobj.getHash()
|
||||||
fileuuid = str(uuid.uuid4())
|
fileuuid = str(uuid.uuid4())
|
||||||
self._execute("INSERT INTO contentmonster_file(uuid, directory, name, checksum) VALUES (?, ?, ?, ?)",
|
self._execute(
|
||||||
(fileuuid, fileobj.directory.name, fileobj.name, hash))
|
"INSERT INTO contentmonster_file(uuid, directory, name, checksum) VALUES (?, ?, ?, ?)",
|
||||||
|
(fileuuid, fileobj.directory.name, fileobj.name, hash),
|
||||||
|
)
|
||||||
return fileuuid
|
return fileuuid
|
||||||
|
|
||||||
def getFileByUUID(self, fileuuid: str) -> Optional[tuple[str, str, str]]:
|
def getFileByUUID(self, fileuuid: str) -> Optional[tuple[str, str, str]]:
|
||||||
|
@ -129,20 +147,24 @@ class Database:
|
||||||
"""
|
"""
|
||||||
cur = self.getCursor()
|
cur = self.getCursor()
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"SELECT directory, name, checksum FROM contentmonster_file WHERE uuid = ?", (fileuuid,))
|
"SELECT directory, name, checksum FROM contentmonster_file WHERE uuid = ?",
|
||||||
if (result := cur.fetchone()):
|
(fileuuid,),
|
||||||
|
)
|
||||||
|
if result := cur.fetchone():
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def removeFile(self, directory, name: str) -> None:
|
def removeFile(self, directory, name: str) -> None:
|
||||||
"""Remove a File from the database based on Directory and filename
|
"""Remove a File from the database based on Directory and filename
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
directory (classes.directory.Directory): Directory object
|
directory (classes.directory.Directory): Directory object
|
||||||
containing the File to remove
|
containing the File to remove
|
||||||
name (str): Filename of the File to remove
|
name (str): Filename of the File to remove
|
||||||
"""
|
"""
|
||||||
self._execute(
|
self._execute(
|
||||||
"DELETE FROM contentmonster_file WHERE directory = ? AND name = ?", (directory.name, name))
|
"DELETE FROM contentmonster_file WHERE directory = ? AND name = ?",
|
||||||
|
(directory.name, name),
|
||||||
|
)
|
||||||
|
|
||||||
def removeFileByUUID(self, fileuuid: str) -> None:
|
def removeFileByUUID(self, fileuuid: str) -> None:
|
||||||
"""Remove a File from the database based on UUID
|
"""Remove a File from the database based on UUID
|
||||||
|
@ -150,19 +172,20 @@ class Database:
|
||||||
Args:
|
Args:
|
||||||
fileuuid (str): The UUID of the File to remove from the database
|
fileuuid (str): The UUID of the File to remove from the database
|
||||||
"""
|
"""
|
||||||
self._execute(
|
self._execute("DELETE FROM contentmonster_file WHERE uuid = ?", (fileuuid,))
|
||||||
"DELETE FROM contentmonster_file WHERE uuid = ?", (fileuuid,))
|
|
||||||
|
|
||||||
def logCompletion(self, file, vessel):
|
def logCompletion(self, file, vessel):
|
||||||
"""Log the completion of a File upload
|
"""Log the completion of a File upload
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
file (classes.file.File): The File object that has been uploaded
|
file (classes.file.File): The File object that has been uploaded
|
||||||
vessel (classes.vessel.Vessel): The Vessel the File has been
|
vessel (classes.vessel.Vessel): The Vessel the File has been
|
||||||
uploaded to
|
uploaded to
|
||||||
"""
|
"""
|
||||||
self._execute(
|
self._execute(
|
||||||
"INSERT INTO contentmonster_file_log(file, vessel) VALUES(?, ?)", (file.uuid, vessel.name))
|
"INSERT INTO contentmonster_file_log(file, vessel) VALUES(?, ?)",
|
||||||
|
(file.uuid, vessel.name),
|
||||||
|
)
|
||||||
|
|
||||||
def getCompletionForVessel(self, vessel) -> list[Optional[str]]:
|
def getCompletionForVessel(self, vessel) -> list[Optional[str]]:
|
||||||
"""Get completed uploads for a vessel
|
"""Get completed uploads for a vessel
|
||||||
|
@ -176,37 +199,44 @@ class Database:
|
||||||
"""
|
"""
|
||||||
cur = self.getCursor()
|
cur = self.getCursor()
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"SELECT file FROM contentmonster_file_log WHERE vessel = ?", (vessel.name,))
|
"SELECT file FROM contentmonster_file_log WHERE vessel = ?", (vessel.name,)
|
||||||
|
)
|
||||||
|
|
||||||
return [f[0] for f in cur.fetchall()]
|
return [f[0] for f in cur.fetchall()]
|
||||||
|
|
||||||
def getCompletionByFileUUID(self, fileuuid: str) -> list[Optional[str]]:
|
def getCompletionByFileUUID(self, fileuuid: str) -> list[Optional[str]]:
|
||||||
cur = self.getCursor()
|
cur = self.getCursor()
|
||||||
cur.execute("SELECT vessel FROM contentmonster_file_log WHERE file = ?", (fileuuid,))
|
cur.execute(
|
||||||
|
"SELECT vessel FROM contentmonster_file_log WHERE file = ?", (fileuuid,)
|
||||||
|
)
|
||||||
|
|
||||||
return [v[0] for v in cur.fetchall()]
|
return [v[0] for v in cur.fetchall()]
|
||||||
|
|
||||||
def migrate(self) -> None:
|
def migrate(self) -> None:
|
||||||
"""Apply database migrations
|
"""Apply database migrations"""
|
||||||
"""
|
|
||||||
cur = self.getCursor()
|
cur = self.getCursor()
|
||||||
|
|
||||||
if self.getVersion() == 0:
|
if self.getVersion() == 0:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"CREATE TABLE IF NOT EXISTS contentmonster_settings(key VARCHAR(64) PRIMARY KEY, value TEXT)")
|
"CREATE TABLE IF NOT EXISTS contentmonster_settings(key VARCHAR(64) PRIMARY KEY, value TEXT)"
|
||||||
|
)
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"INSERT INTO contentmonster_settings(key, value) VALUES ('dbversion', '1')")
|
"INSERT INTO contentmonster_settings(key, value) VALUES ('dbversion', '1')"
|
||||||
|
)
|
||||||
self.commit()
|
self.commit()
|
||||||
|
|
||||||
if self.getVersion() == 1:
|
if self.getVersion() == 1:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"CREATE TABLE IF NOT EXISTS contentmonster_file(uuid VARCHAR(36) PRIMARY KEY, directory VARCHAR(128), name VARCHAR(128), checksum VARCHAR(64))")
|
"CREATE TABLE IF NOT EXISTS contentmonster_file(uuid VARCHAR(36) PRIMARY KEY, directory VARCHAR(128), name VARCHAR(128), checksum VARCHAR(64))"
|
||||||
cur.execute("CREATE TABLE IF NOT EXISTS contentmonster_file_log(file VARCHAR(36), vessel VARCHAR(128), PRIMARY KEY (file, vessel), FOREIGN KEY (file) REFERENCES contentmonster_files(uuid) ON DELETE CASCADE)")
|
)
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"UPDATE contentmonster_settings SET value = '2' WHERE key = 'dbversion'")
|
"CREATE TABLE IF NOT EXISTS contentmonster_file_log(file VARCHAR(36), vessel VARCHAR(128), PRIMARY KEY (file, vessel), FOREIGN KEY (file) REFERENCES contentmonster_files(uuid) ON DELETE CASCADE)"
|
||||||
|
)
|
||||||
|
cur.execute(
|
||||||
|
"UPDATE contentmonster_settings SET value = '2' WHERE key = 'dbversion'"
|
||||||
|
)
|
||||||
self.commit()
|
self.commit()
|
||||||
|
|
||||||
def __del__(self):
|
def __del__(self):
|
||||||
"""Close database connection on removal of the Database object
|
"""Close database connection on removal of the Database object"""
|
||||||
"""
|
|
||||||
self._con.close()
|
self._con.close()
|
||||||
|
|
|
@ -8,14 +8,14 @@ from typing import Union, Optional
|
||||||
|
|
||||||
|
|
||||||
class Directory:
|
class Directory:
|
||||||
"""Class representing a Directory on the local filesystem
|
"""Class representing a Directory on the local filesystem"""
|
||||||
"""
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def fromConfig(cls, config: SectionProxy):
|
def fromConfig(cls, config: SectionProxy):
|
||||||
"""Create Directory object from a Directory section in the Config file
|
"""Create Directory object from a Directory section in the Config file
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config (configparser.SectionProxy): Configuration section defining
|
config (configparser.SectionProxy): Configuration section defining
|
||||||
a Directory
|
a Directory
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
|
@ -28,8 +28,11 @@ class Directory:
|
||||||
if "Location" in config.keys():
|
if "Location" in config.keys():
|
||||||
return cls(config.name.split()[1], config["Location"])
|
return cls(config.name.split()[1], config["Location"])
|
||||||
else:
|
else:
|
||||||
raise ValueError("Definition for Directory " +
|
raise ValueError(
|
||||||
config.name.split()[1] + " does not contain Location!")
|
"Definition for Directory "
|
||||||
|
+ config.name.split()[1]
|
||||||
|
+ " does not contain Location!"
|
||||||
|
)
|
||||||
|
|
||||||
def __init__(self, name: str, location: Union[str, pathlib.Path]):
|
def __init__(self, name: str, location: Union[str, pathlib.Path]):
|
||||||
"""Initialize a new Directory object
|
"""Initialize a new Directory object
|
||||||
|
@ -39,7 +42,7 @@ class Directory:
|
||||||
location (str, pathlib.Path): Filesystem location of the Directory
|
location (str, pathlib.Path): Filesystem location of the Directory
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
ValueError: Raised if passed location does not exist or is not a
|
ValueError: Raised if passed location does not exist or is not a
|
||||||
directory
|
directory
|
||||||
"""
|
"""
|
||||||
self.name = name
|
self.name = name
|
||||||
|
@ -50,7 +53,8 @@ class Directory:
|
||||||
else:
|
else:
|
||||||
location = str(location)
|
location = str(location)
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Location {location} for Directory {name} does not exist or is not a directory.")
|
f"Location {location} for Directory {name} does not exist or is not a directory."
|
||||||
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def completeddir(self):
|
def completeddir(self):
|
||||||
|
@ -59,7 +63,10 @@ class Directory:
|
||||||
def assertCompletedDirectory(self):
|
def assertCompletedDirectory(self):
|
||||||
if not os.path.isdir(self.completeddir):
|
if not os.path.isdir(self.completeddir):
|
||||||
if os.path.isfile(self.completeddir):
|
if os.path.isfile(self.completeddir):
|
||||||
raise FileExistsError("Cannot create directory %s - path exists but is not a directory!" % str(self.completeddir))
|
raise FileExistsError(
|
||||||
|
"Cannot create directory %s - path exists but is not a directory!"
|
||||||
|
% str(self.completeddir)
|
||||||
|
)
|
||||||
|
|
||||||
os.mkdir(self.completeddir)
|
os.mkdir(self.completeddir)
|
||||||
|
|
||||||
|
@ -71,8 +78,11 @@ class Directory:
|
||||||
Returns:
|
Returns:
|
||||||
list: List of File objects for files within the Directory
|
list: List of File objects for files within the Directory
|
||||||
"""
|
"""
|
||||||
files = [f for f in os.listdir(self.location) if os.path.isfile(
|
files = [
|
||||||
self.location / f) and os.path.getsize(self.location / f)]
|
f
|
||||||
|
for f in os.listdir(self.location)
|
||||||
|
if os.path.isfile(self.location / f) and os.path.getsize(self.location / f)
|
||||||
|
]
|
||||||
return [File(f, self) for f in files]
|
return [File(f, self) for f in files]
|
||||||
|
|
||||||
def getFile(self, name: str) -> Optional[File]:
|
def getFile(self, name: str) -> Optional[File]:
|
||||||
|
|
|
@ -1,6 +1,11 @@
|
||||||
from watchdog.events import (FileSystemEventHandler, FileSystemEvent,
|
from watchdog.events import (
|
||||||
FileCreatedEvent, FileDeletedEvent,
|
FileSystemEventHandler,
|
||||||
FileModifiedEvent, FileMovedEvent)
|
FileSystemEvent,
|
||||||
|
FileCreatedEvent,
|
||||||
|
FileDeletedEvent,
|
||||||
|
FileModifiedEvent,
|
||||||
|
FileMovedEvent,
|
||||||
|
)
|
||||||
|
|
||||||
from multiprocessing import Queue
|
from multiprocessing import Queue
|
||||||
|
|
||||||
|
@ -11,8 +16,7 @@ import time
|
||||||
|
|
||||||
|
|
||||||
class DogHandler(FileSystemEventHandler):
|
class DogHandler(FileSystemEventHandler):
|
||||||
"""Class implementing a watchdog event handler
|
"""Class implementing a watchdog event handler"""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, directory, queue: Queue, *args, **kwargs) -> None:
|
def __init__(self, directory, queue: Queue, *args, **kwargs) -> None:
|
||||||
"""Initialize a new DogHandler object
|
"""Initialize a new DogHandler object
|
||||||
|
@ -71,7 +75,8 @@ class DogHandler(FileSystemEventHandler):
|
||||||
file (source and destination)
|
file (source and destination)
|
||||||
"""
|
"""
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"Detected move event of {event.src_path} to {event.dest_path}")
|
f"Detected move event of {event.src_path} to {event.dest_path}"
|
||||||
|
)
|
||||||
self._queue.put((self._directory, os.path.basename(event.src_path)))
|
self._queue.put((self._directory, os.path.basename(event.src_path)))
|
||||||
self._queue.put((self._directory, os.path.basename(event.dest_path)))
|
self._queue.put((self._directory, os.path.basename(event.dest_path)))
|
||||||
|
|
||||||
|
|
|
@ -8,10 +8,11 @@ import os.path
|
||||||
|
|
||||||
|
|
||||||
class File:
|
class File:
|
||||||
"""Object representing a file found in a local Directory
|
"""Object representing a file found in a local Directory"""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, name: str, directory, uuid: Optional[str] = None, dbclass: type = Database) -> None:
|
def __init__(
|
||||||
|
self, name: str, directory, uuid: Optional[str] = None, dbclass: type = Database
|
||||||
|
) -> None:
|
||||||
"""Initialize new File object
|
"""Initialize new File object
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
@ -31,7 +32,9 @@ class File:
|
||||||
self.dbclass = dbclass
|
self.dbclass = dbclass
|
||||||
|
|
||||||
if not self.exists():
|
if not self.exists():
|
||||||
raise FileNotFoundError(f"File {self.name} does not exist in {self.directory.name}!")
|
raise FileNotFoundError(
|
||||||
|
f"File {self.name} does not exist in {self.directory.name}!"
|
||||||
|
)
|
||||||
|
|
||||||
self.uuid = uuid or self.getUUID()
|
self.uuid = uuid or self.getUUID()
|
||||||
|
|
||||||
|
@ -90,7 +93,8 @@ class File:
|
||||||
"""
|
"""
|
||||||
if count != -1 and not size:
|
if count != -1 and not size:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"A Chunk size needs to be passed to getChunk() if not using the complete file (-1)!")
|
"A Chunk size needs to be passed to getChunk() if not using the complete file (-1)!"
|
||||||
|
)
|
||||||
|
|
||||||
with open(self.getFullPath(), "rb") as binary:
|
with open(self.getFullPath(), "rb") as binary:
|
||||||
binary.seek((count * size) if count > 0 else 0)
|
binary.seek((count * size) if count > 0 else 0)
|
||||||
|
|
|
@ -3,4 +3,7 @@ from logging import Logger as PyLogger
|
||||||
|
|
||||||
class Logger(PyLogger):
|
class Logger(PyLogger):
|
||||||
def __init__(self, name="contentmonster"):
|
def __init__(self, name="contentmonster"):
|
||||||
super().__init__(name)
|
super().__init__(name)
|
||||||
|
|
||||||
|
def trace(self, msg, *args, **kwargs) -> None:
|
||||||
|
pass
|
||||||
|
|
|
@ -2,8 +2,7 @@ from ..const import STATUS_COMPLETE, STATUS_START
|
||||||
|
|
||||||
|
|
||||||
class RemoteFile:
|
class RemoteFile:
|
||||||
"""Class describing the transfer status of a File to a Vessel
|
"""Class describing the transfer status of a File to a Vessel"""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, fileobj, vessel, chunksize: int) -> None:
|
def __init__(self, fileobj, vessel, chunksize: int) -> None:
|
||||||
"""Initialize a new RemoteFile object
|
"""Initialize a new RemoteFile object
|
||||||
|
@ -29,8 +28,7 @@ class RemoteFile:
|
||||||
# Get all files in the vessel's tempdir
|
# Get all files in the vessel's tempdir
|
||||||
|
|
||||||
ls = self.vessel.connection._listdir(self.vessel.tempdir)
|
ls = self.vessel.connection._listdir(self.vessel.tempdir)
|
||||||
files = [f for f in ls if f.startswith(
|
files = [f for f in ls if f.startswith(self.file.uuid) and f.endswith(".part")]
|
||||||
self.file.uuid) and f.endswith(".part")]
|
|
||||||
|
|
||||||
# Find the file with the largest chunk number
|
# Find the file with the largest chunk number
|
||||||
|
|
||||||
|
@ -39,11 +37,14 @@ class RemoteFile:
|
||||||
for f in files:
|
for f in files:
|
||||||
part = f.split("_")[1].split(".")[0]
|
part = f.split("_")[1].split(".")[0]
|
||||||
if part == "complete": # If a reassembled file is found
|
if part == "complete": # If a reassembled file is found
|
||||||
if self.validateComplete(True): # and it is not broken
|
if self.validateComplete(True): # and it is not broken
|
||||||
return STATUS_COMPLETE # the upload is complete
|
return STATUS_COMPLETE # the upload is complete
|
||||||
|
|
||||||
# Else save the chunk number if it is larger than the previous
|
# Else save the chunk number if it is larger than the previous
|
||||||
count = max(count, int(part))
|
try:
|
||||||
|
count = max(count, int(part))
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
# Find and return the largest non-corrupt chunk
|
# Find and return the largest non-corrupt chunk
|
||||||
while count >= 0:
|
while count >= 0:
|
||||||
|
@ -80,8 +81,7 @@ class RemoteFile:
|
||||||
return self.vessel.connection.assertComplete(self, allow_retry)
|
return self.vessel.connection.assertComplete(self, allow_retry)
|
||||||
|
|
||||||
def compileComplete(self) -> None:
|
def compileComplete(self) -> None:
|
||||||
"""Reassemble a complete File from the uploaded Chunks
|
"""Reassemble a complete File from the uploaded Chunks"""
|
||||||
"""
|
|
||||||
self.vessel.connection.compileComplete(self)
|
self.vessel.connection.compileComplete(self)
|
||||||
|
|
||||||
def getChunk(self, count: int):
|
def getChunk(self, count: int):
|
||||||
|
@ -91,7 +91,7 @@ class RemoteFile:
|
||||||
count (int): Number of the Chunk to generate
|
count (int): Number of the Chunk to generate
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
classes.chunk.Chunk: A Chunk object containing the portion of the
|
classes.chunk.Chunk: A Chunk object containing the portion of the
|
||||||
File object beginning at (count * chunksize) bytes and ending at
|
File object beginning at (count * chunksize) bytes and ending at
|
||||||
((count + 1) * chunksize - 1) bytes, with chunksize taken from the
|
((count + 1) * chunksize - 1) bytes, with chunksize taken from the
|
||||||
RemoteFile initialization value
|
RemoteFile initialization value
|
||||||
|
@ -99,7 +99,6 @@ class RemoteFile:
|
||||||
return self.file.getChunk(count, self.chunksize)
|
return self.file.getChunk(count, self.chunksize)
|
||||||
|
|
||||||
def finalizeUpload(self) -> None:
|
def finalizeUpload(self) -> None:
|
||||||
"""Move complete file to its final destination and clean up
|
"""Move complete file to its final destination and clean up"""
|
||||||
"""
|
|
||||||
self.vessel.connection.moveComplete(self)
|
self.vessel.connection.moveComplete(self)
|
||||||
self.vessel.connection.clearTempDir()
|
self.vessel.connection.clearTempDir()
|
||||||
|
|
|
@ -5,8 +5,8 @@ from .logger import Logger
|
||||||
|
|
||||||
|
|
||||||
class retry:
|
class retry:
|
||||||
"""Decorator used to automatically retry operations throwing exceptions
|
"""Decorator used to automatically retry operations throwing exceptions"""
|
||||||
"""
|
|
||||||
def __init__(self, exceptions: tuple[BaseException] = None):
|
def __init__(self, exceptions: tuple[BaseException] = None):
|
||||||
"""Initializing the retry decorator
|
"""Initializing the retry decorator
|
||||||
|
|
||||||
|
@ -16,10 +16,14 @@ class retry:
|
||||||
paramiko.ssh_exception.SSHException/NoValidConnectionsError and
|
paramiko.ssh_exception.SSHException/NoValidConnectionsError and
|
||||||
socket.timeout/TimeoutError. Defaults to None.
|
socket.timeout/TimeoutError. Defaults to None.
|
||||||
"""
|
"""
|
||||||
self.exceptions = exceptions or (SSHException, NoValidConnectionsError,
|
self.exceptions = exceptions or (
|
||||||
timeout, TimeoutError)
|
SSHException,
|
||||||
|
NoValidConnectionsError,
|
||||||
|
timeout,
|
||||||
|
TimeoutError,
|
||||||
|
)
|
||||||
self._logger = Logger()
|
self._logger = Logger()
|
||||||
|
|
||||||
def __call__(self, f):
|
def __call__(self, f):
|
||||||
"""Return a function through the retry decorator
|
"""Return a function through the retry decorator
|
||||||
|
|
||||||
|
@ -29,6 +33,7 @@ class retry:
|
||||||
Returns:
|
Returns:
|
||||||
function: Function wrapping the passed function
|
function: Function wrapping the passed function
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def wrapped_f(*args, **kwargs):
|
def wrapped_f(*args, **kwargs):
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
|
@ -36,4 +41,4 @@ class retry:
|
||||||
except self.exceptions as e:
|
except self.exceptions as e:
|
||||||
self._logger.info("Caught expected exception: " + repr(e))
|
self._logger.info("Caught expected exception: " + repr(e))
|
||||||
|
|
||||||
return wrapped_f
|
return wrapped_f
|
||||||
|
|
|
@ -15,8 +15,8 @@ import os.path
|
||||||
|
|
||||||
|
|
||||||
class ShoreThread(Process):
|
class ShoreThread(Process):
|
||||||
"""Thread handling the discovery of shore-side file changes
|
"""Thread handling the discovery of shore-side file changes"""
|
||||||
"""
|
|
||||||
def __init__(self, state: dict, dbclass: type = Database) -> None:
|
def __init__(self, state: dict, dbclass: type = Database) -> None:
|
||||||
"""Create a new ShoreThread object
|
"""Create a new ShoreThread object
|
||||||
|
|
||||||
|
@ -48,14 +48,12 @@ class ShoreThread(Process):
|
||||||
return files
|
return files
|
||||||
|
|
||||||
def clearFiles(self) -> None:
|
def clearFiles(self) -> None:
|
||||||
"""Clear the files variable in the application state
|
"""Clear the files variable in the application state"""
|
||||||
"""
|
|
||||||
self._logger.debug("Clearing global files variable")
|
self._logger.debug("Clearing global files variable")
|
||||||
del self._state["files"][:]
|
del self._state["files"][:]
|
||||||
|
|
||||||
def monitor(self) -> None:
|
def monitor(self) -> None:
|
||||||
"""Initialize monitoring of Directories specified in configuration
|
"""Initialize monitoring of Directories specified in configuration"""
|
||||||
"""
|
|
||||||
for directory in self._state["config"].directories:
|
for directory in self._state["config"].directories:
|
||||||
self._logger.debug("Creating dog for " + str(directory.location))
|
self._logger.debug("Creating dog for " + str(directory.location))
|
||||||
handler = DogHandler(directory, self.queue)
|
handler = DogHandler(directory, self.queue)
|
||||||
|
@ -65,8 +63,7 @@ class ShoreThread(Process):
|
||||||
self._dogs.append(dog)
|
self._dogs.append(dog)
|
||||||
|
|
||||||
def run(self) -> NoReturn:
|
def run(self) -> NoReturn:
|
||||||
"""Launch the ShoreThread and start monitoring for file changes
|
"""Launch the ShoreThread and start monitoring for file changes"""
|
||||||
"""
|
|
||||||
self._logger.info("Launched Shore Thread")
|
self._logger.info("Launched Shore Thread")
|
||||||
|
|
||||||
for f in self.getAllFiles():
|
for f in self.getAllFiles():
|
||||||
|
@ -79,8 +76,7 @@ class ShoreThread(Process):
|
||||||
self.processQueue()
|
self.processQueue()
|
||||||
|
|
||||||
def joinDogs(self) -> None:
|
def joinDogs(self) -> None:
|
||||||
"""Join observers to receive updates on the queue
|
"""Join observers to receive updates on the queue"""
|
||||||
"""
|
|
||||||
self._logger.debug("Joining dogs")
|
self._logger.debug("Joining dogs")
|
||||||
for dog in self._dogs:
|
for dog in self._dogs:
|
||||||
dog.join(1)
|
dog.join(1)
|
||||||
|
@ -103,7 +99,7 @@ class ShoreThread(Process):
|
||||||
outlist.append(f)
|
outlist.append(f)
|
||||||
|
|
||||||
self.clearFiles()
|
self.clearFiles()
|
||||||
|
|
||||||
for f in outlist:
|
for f in outlist:
|
||||||
self._state["files"].append(f)
|
self._state["files"].append(f)
|
||||||
|
|
||||||
|
@ -118,7 +114,9 @@ class ShoreThread(Process):
|
||||||
Args:
|
Args:
|
||||||
fileobj (classes.file.File): File object to add to the queue
|
fileobj (classes.file.File): File object to add to the queue
|
||||||
"""
|
"""
|
||||||
self._logger.debug(f"Adding file {fileobj.name} to directory {fileobj.directory.name}")
|
self._logger.debug(
|
||||||
|
f"Adding file {fileobj.name} to directory {fileobj.directory.name}"
|
||||||
|
)
|
||||||
|
|
||||||
outlist = []
|
outlist = []
|
||||||
|
|
||||||
|
@ -133,9 +131,13 @@ class ShoreThread(Process):
|
||||||
outlist.append(f)
|
outlist.append(f)
|
||||||
|
|
||||||
if self.checkFileCompletion(fileobj):
|
if self.checkFileCompletion(fileobj):
|
||||||
self._logger.debug(f"File {fileobj.name} already transferred to all Vessels - not re-adding to queue")
|
self._logger.debug(
|
||||||
|
f"File {fileobj.name} already transferred to all Vessels - not re-adding to queue"
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
self._logger.debug(f"File {fileobj.name} not in processing queue (anymore) - adding")
|
self._logger.debug(
|
||||||
|
f"File {fileobj.name} not in processing queue (anymore) - adding"
|
||||||
|
)
|
||||||
outlist.append(fileobj)
|
outlist.append(fileobj)
|
||||||
|
|
||||||
self.clearFiles()
|
self.clearFiles()
|
||||||
|
@ -146,15 +148,18 @@ class ShoreThread(Process):
|
||||||
def checkFileCompletion(self, fileobj: File) -> bool:
|
def checkFileCompletion(self, fileobj: File) -> bool:
|
||||||
db = self._dbclass()
|
db = self._dbclass()
|
||||||
complete = db.getCompletionByFileUUID(fileobj.uuid)
|
complete = db.getCompletionByFileUUID(fileobj.uuid)
|
||||||
del(db)
|
del db
|
||||||
|
|
||||||
for vessel in self._state["config"].vessels:
|
for vessel in self._state["config"].vessels:
|
||||||
if vessel.name not in complete and fileobj.directory.name not in vessel._ignoredirs:
|
if (
|
||||||
|
vessel.name not in complete
|
||||||
|
and fileobj.directory.name not in vessel._ignoredirs
|
||||||
|
):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if fileobj.exists():
|
if fileobj.exists():
|
||||||
fileobj.moveCompleted()
|
fileobj.moveCompleted()
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def processFile(self, directory: Directory, name: str) -> None:
|
def processFile(self, directory: Directory, name: str) -> None:
|
||||||
|
@ -163,11 +168,13 @@ class ShoreThread(Process):
|
||||||
Args:
|
Args:
|
||||||
directory (classes.directory.Directory): Directory containing the
|
directory (classes.directory.Directory): Directory containing the
|
||||||
created, deleted, modified or moved File
|
created, deleted, modified or moved File
|
||||||
name (str): Filename of the created, deleted, modified or moved
|
name (str): Filename of the created, deleted, modified or moved
|
||||||
File
|
File
|
||||||
"""
|
"""
|
||||||
self._logger.debug(f"Processing change to file {name} in directory {directory.name}")
|
self._logger.debug(
|
||||||
if (fileobj := directory.getFile(name)):
|
f"Processing change to file {name} in directory {directory.name}"
|
||||||
|
)
|
||||||
|
if fileobj := directory.getFile(name):
|
||||||
self.addFile(fileobj)
|
self.addFile(fileobj)
|
||||||
else:
|
else:
|
||||||
self.purgeFile(directory, name)
|
self.purgeFile(directory, name)
|
||||||
|
@ -180,12 +187,11 @@ class ShoreThread(Process):
|
||||||
File that has been created, moved, modified or deleted.
|
File that has been created, moved, modified or deleted.
|
||||||
"""
|
"""
|
||||||
self._logger.debug("Waiting for new changes...")
|
self._logger.debug("Waiting for new changes...")
|
||||||
directory, basename = self.queue.get() # Will block until an event is found
|
directory, basename = self.queue.get() # Will block until an event is found
|
||||||
self.processFile(directory, basename)
|
self.processFile(directory, basename)
|
||||||
|
|
||||||
def terminate(self, *args, **kwargs) -> NoReturn:
|
def terminate(self, *args, **kwargs) -> NoReturn:
|
||||||
"""Terminate observer threads, then terminate self
|
"""Terminate observer threads, then terminate self"""
|
||||||
"""
|
|
||||||
self._logger.info("Terminating dogs and shore thread")
|
self._logger.info("Terminating dogs and shore thread")
|
||||||
for dog in self._dogs:
|
for dog in self._dogs:
|
||||||
dog.terminate()
|
dog.terminate()
|
||||||
|
|
|
@ -11,14 +11,14 @@ import pathlib
|
||||||
|
|
||||||
|
|
||||||
class Vessel:
|
class Vessel:
|
||||||
"""Class describing a Vessel (= a replication destination)
|
"""Class describing a Vessel (= a replication destination)"""
|
||||||
"""
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def fromConfig(cls, config: SectionProxy, dbclass: type = Database):
|
def fromConfig(cls, config: SectionProxy, dbclass: type = Database):
|
||||||
"""Create Vessel object from a Vessel section in the Config file
|
"""Create Vessel object from a Vessel section in the Config file
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
config (configparser.SectionProxy): Vessel section defining a
|
config (configparser.SectionProxy): Vessel section defining a
|
||||||
Vessel
|
Vessel
|
||||||
dbclass (type): Class to use for database connections. Defaults to
|
dbclass (type): Class to use for database connections. Defaults to
|
||||||
built-in Database using sqlite3.
|
built-in Database using sqlite3.
|
||||||
|
@ -61,17 +61,38 @@ class Vessel:
|
||||||
ignoredirs = [d.strip() for d in config["IgnoreDirs"].split(",")]
|
ignoredirs = [d.strip() for d in config["IgnoreDirs"].split(",")]
|
||||||
|
|
||||||
if "Address" in config.keys():
|
if "Address" in config.keys():
|
||||||
return cls(config.name.split()[1], config["Address"], username,
|
return cls(
|
||||||
password, passphrase, port, timeout, tempdir, ignoredirs, dbclass)
|
config.name.split()[1],
|
||||||
|
config["Address"],
|
||||||
|
username,
|
||||||
|
password,
|
||||||
|
passphrase,
|
||||||
|
port,
|
||||||
|
timeout,
|
||||||
|
tempdir,
|
||||||
|
ignoredirs,
|
||||||
|
dbclass,
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
raise ValueError("Definition for Vessel " +
|
raise ValueError(
|
||||||
config.name.split()[1] + " does not contain Address!")
|
"Definition for Vessel "
|
||||||
|
+ config.name.split()[1]
|
||||||
|
+ " does not contain Address!"
|
||||||
|
)
|
||||||
|
|
||||||
def __init__(self, name: str, address: str, username: Optional[str] = None,
|
def __init__(
|
||||||
password: Optional[str] = None, passphrase: Optional[str] = None,
|
self,
|
||||||
port: Optional[int] = None, timeout: Optional[int] = None,
|
name: str,
|
||||||
tempdir: Optional[Union[str, pathlib.Path]] = None,
|
address: str,
|
||||||
ignoredirs: list[Optional[str]] = [], dbclass: type = Database) -> None:
|
username: Optional[str] = None,
|
||||||
|
password: Optional[str] = None,
|
||||||
|
passphrase: Optional[str] = None,
|
||||||
|
port: Optional[int] = None,
|
||||||
|
timeout: Optional[int] = None,
|
||||||
|
tempdir: Optional[Union[str, pathlib.Path]] = None,
|
||||||
|
ignoredirs: list[Optional[str]] = [],
|
||||||
|
dbclass: type = Database,
|
||||||
|
) -> None:
|
||||||
"""Initialize new Vessel object
|
"""Initialize new Vessel object
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
@ -141,8 +162,7 @@ class Vessel:
|
||||||
return output
|
return output
|
||||||
|
|
||||||
def clearTempDir(self) -> None:
|
def clearTempDir(self) -> None:
|
||||||
"""Clean up the temporary directory on the Vessel
|
"""Clean up the temporary directory on the Vessel"""
|
||||||
"""
|
|
||||||
self.connection.clearTempDir()
|
self.connection.clearTempDir()
|
||||||
|
|
||||||
def pushChunk(self, chunk, path: Optional[Union[str, pathlib.Path]] = None) -> None:
|
def pushChunk(self, chunk, path: Optional[Union[str, pathlib.Path]] = None) -> None:
|
||||||
|
@ -165,7 +185,7 @@ class Vessel:
|
||||||
remotefile (classes.remotefile.RemoteFile): RemoteFile object
|
remotefile (classes.remotefile.RemoteFile): RemoteFile object
|
||||||
describing the uploaded File
|
describing the uploaded File
|
||||||
"""
|
"""
|
||||||
self.connection.compileComplete(remotefile)
|
return self.connection.compileComplete(remotefile)
|
||||||
|
|
||||||
def assertDirectories(self, directory) -> None:
|
def assertDirectories(self, directory) -> None:
|
||||||
"""Make sure that destination and temp directories exist on the Vessel
|
"""Make sure that destination and temp directories exist on the Vessel
|
||||||
|
@ -177,7 +197,7 @@ class Vessel:
|
||||||
Raises:
|
Raises:
|
||||||
ValueError: Raised if a path is already in use on the vessel but
|
ValueError: Raised if a path is already in use on the vessel but
|
||||||
not a directory.
|
not a directory.
|
||||||
IOError: Raised if a directory that does not exist cannot be
|
IOError: Raised if a directory that does not exist cannot be
|
||||||
created.
|
created.
|
||||||
"""
|
"""
|
||||||
self.connection.assertDirectories(directory)
|
self.connection.assertDirectories(directory)
|
||||||
|
@ -188,7 +208,7 @@ class Vessel:
|
||||||
Raises:
|
Raises:
|
||||||
ValueError: Raised if the path is already in use on the vessel but
|
ValueError: Raised if the path is already in use on the vessel but
|
||||||
is not a directory.
|
is not a directory.
|
||||||
IOError: Raised if the directory does not exist but cannot be
|
IOError: Raised if the directory does not exist but cannot be
|
||||||
created.
|
created.
|
||||||
"""
|
"""
|
||||||
self.connection.assertTempDirectory()
|
self.connection.assertTempDirectory()
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
from multiprocessing import Process
|
from multiprocessing import Process
|
||||||
from typing import NoReturn, Optional
|
from typing import NoReturn, Optional
|
||||||
|
from traceback import format_exception
|
||||||
|
|
||||||
from .vessel import Vessel
|
from .vessel import Vessel
|
||||||
from .remotefile import RemoteFile
|
from .remotefile import RemoteFile
|
||||||
|
@ -10,11 +11,11 @@ from .file import File
|
||||||
from ..const import STATUS_COMPLETE, STATUS_START
|
from ..const import STATUS_COMPLETE, STATUS_START
|
||||||
|
|
||||||
import time
|
import time
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
class VesselThread(Process):
|
class VesselThread(Process):
|
||||||
"""Thread processing uploads to a single vessel
|
"""Thread processing uploads to a single vessel"""
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, vessel: Vessel, state: dict, dbclass: type = Database) -> None:
|
def __init__(self, vessel: Vessel, state: dict, dbclass: type = Database) -> None:
|
||||||
"""Initialize a new VesselThread
|
"""Initialize a new VesselThread
|
||||||
|
@ -30,8 +31,7 @@ class VesselThread(Process):
|
||||||
self._dbclass = dbclass
|
self._dbclass = dbclass
|
||||||
|
|
||||||
def run(self) -> NoReturn:
|
def run(self) -> NoReturn:
|
||||||
"""Run thread and process uploads to the vessel
|
"""Run thread and process uploads to the vessel"""
|
||||||
"""
|
|
||||||
self._logger.debug("Launched Vessel Thread for " + self.vessel.name)
|
self._logger.debug("Launched Vessel Thread for " + self.vessel.name)
|
||||||
self.assertDirectories()
|
self.assertDirectories()
|
||||||
while True:
|
while True:
|
||||||
|
@ -39,31 +39,39 @@ class VesselThread(Process):
|
||||||
self.upload()
|
self.upload()
|
||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self._logger.error("An exception occurred in the Vessel Thread for " +
|
exc_type, exc_value, exc_traceback = sys.exc_info()
|
||||||
self.vessel.name)
|
|
||||||
self._logger.error(repr(e))
|
self._logger.error(
|
||||||
|
"An exception occurred in the Vessel Thread for " + self.vessel.name
|
||||||
|
)
|
||||||
|
|
||||||
|
self._logger.error(
|
||||||
|
"\n".join(format_exception(exc_type, exc_value, exc_traceback))
|
||||||
|
)
|
||||||
|
|
||||||
@retry()
|
@retry()
|
||||||
def assertDirectories(self) -> None:
|
def assertDirectories(self) -> None:
|
||||||
for directory in self._state["config"].directories:
|
for directory in self._state["config"].directories:
|
||||||
if not directory.name in self.vessel._ignoredirs:
|
if not directory.name in self.vessel._ignoredirs:
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"Making sure directory {directory.name} exists on Vessel {self.vessel.name}")
|
f"Making sure directory {directory.name} exists on Vessel {self.vessel.name}"
|
||||||
|
)
|
||||||
self.vessel.connection.assertDirectories(directory)
|
self.vessel.connection.assertDirectories(directory)
|
||||||
|
|
||||||
@retry()
|
@retry()
|
||||||
def upload(self) -> None:
|
def upload(self) -> None:
|
||||||
"""Continue uploading process
|
"""Continue uploading process"""
|
||||||
"""
|
|
||||||
if not (current := (self.vessel.currentUpload() or self.processQueue())):
|
if not (current := (self.vessel.currentUpload() or self.processQueue())):
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"No file needs to be uploaded to Vessel {self.vessel.name} at the moment")
|
f"No file needs to be uploaded to Vessel {self.vessel.name} at the moment"
|
||||||
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
if isinstance(current, tuple):
|
if isinstance(current, tuple):
|
||||||
dirname, name, _ = current
|
dirname, name, _ = current
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"Found file {name} in directory {dirname} for vessel {self.vessel.name}")
|
f"Found file {name} in directory {dirname} for vessel {self.vessel.name}"
|
||||||
|
)
|
||||||
|
|
||||||
directory = None
|
directory = None
|
||||||
|
|
||||||
|
@ -74,7 +82,8 @@ class VesselThread(Process):
|
||||||
|
|
||||||
if not directory:
|
if not directory:
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"Directory {dirname} not specified in config - deleting File from Vessel {self.vessel.name}")
|
f"Directory {dirname} not specified in config - deleting File from Vessel {self.vessel.name}"
|
||||||
|
)
|
||||||
self.vessel.clearTempDir()
|
self.vessel.clearTempDir()
|
||||||
return
|
return
|
||||||
|
|
||||||
|
@ -82,26 +91,28 @@ class VesselThread(Process):
|
||||||
fileobj = File(name, directory)
|
fileobj = File(name, directory)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"File {name} does not exist in Directory {dirname} on shore - deleting from Vessel {self.name}")
|
f"File {name} does not exist in Directory {dirname} on shore - deleting from Vessel {self.name}"
|
||||||
|
)
|
||||||
self.vessel.clearTempDir()
|
self.vessel.clearTempDir()
|
||||||
return
|
return
|
||||||
|
|
||||||
else:
|
else:
|
||||||
fileobj = current
|
fileobj = current
|
||||||
|
|
||||||
remotefile = RemoteFile(fileobj, self.vessel,
|
remotefile = RemoteFile(fileobj, self.vessel, self._state["config"].chunksize)
|
||||||
self._state["config"].chunksize)
|
|
||||||
|
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"Start processing file {fileobj.name} in directory {fileobj.directory.name} on vessel {self.vessel.name}")
|
f"Start processing file {fileobj.name} in directory {fileobj.directory.name} on vessel {self.vessel.name}"
|
||||||
|
)
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
db = self._dbclass()
|
db = self._dbclass()
|
||||||
if not db.getFileByUUID(fileobj.uuid):
|
if not db.getFileByUUID(fileobj.uuid):
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"File {fileobj.name} in directory {fileobj.directory.name} does not exist anymore - deleting from {self.vessel.name}")
|
f"File {fileobj.name} in directory {fileobj.directory.name} does not exist anymore - deleting from {self.vessel.name}"
|
||||||
|
)
|
||||||
self.vessel.clearTempDir()
|
self.vessel.clearTempDir()
|
||||||
del(db)
|
del db
|
||||||
|
|
||||||
self.vessel.assertDirectories(fileobj.directory)
|
self.vessel.assertDirectories(fileobj.directory)
|
||||||
|
|
||||||
|
@ -109,16 +120,18 @@ class VesselThread(Process):
|
||||||
|
|
||||||
if status == STATUS_COMPLETE:
|
if status == STATUS_COMPLETE:
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"File {fileobj.name} uploaded to vessel {self.vessel.name} completely - finalizing")
|
f"File {fileobj.name} uploaded to vessel {self.vessel.name} completely - finalizing"
|
||||||
|
)
|
||||||
remotefile.finalizeUpload()
|
remotefile.finalizeUpload()
|
||||||
|
|
||||||
db = self._dbclass()
|
db = self._dbclass()
|
||||||
db.logCompletion(fileobj, self.vessel)
|
db.logCompletion(fileobj, self.vessel)
|
||||||
del(db)
|
del db
|
||||||
|
|
||||||
self.vessel._uploaded.append(fileobj.uuid)
|
self.vessel._uploaded.append(fileobj.uuid)
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"Moved {fileobj.name} to its final destination on {self.vessel.name} - done!")
|
f"Moved {fileobj.name} to its final destination on {self.vessel.name} - done!"
|
||||||
|
)
|
||||||
|
|
||||||
self.checkFileCompletion(fileobj)
|
self.checkFileCompletion(fileobj)
|
||||||
return
|
return
|
||||||
|
@ -126,7 +139,8 @@ class VesselThread(Process):
|
||||||
nextchunk = 0 if status == STATUS_START else status + 1
|
nextchunk = 0 if status == STATUS_START else status + 1
|
||||||
|
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"Getting chunk #{nextchunk} for file {fileobj.name} for vessel {self.vessel.name}")
|
f"Getting chunk #{nextchunk} for file {fileobj.name} for vessel {self.vessel.name}"
|
||||||
|
)
|
||||||
chunk = remotefile.getChunk(nextchunk)
|
chunk = remotefile.getChunk(nextchunk)
|
||||||
|
|
||||||
self._logger.debug("Got chunk")
|
self._logger.debug("Got chunk")
|
||||||
|
@ -135,45 +149,50 @@ class VesselThread(Process):
|
||||||
# of the file, i.e. the complete file has already been uploaded
|
# of the file, i.e. the complete file has already been uploaded
|
||||||
|
|
||||||
if chunk.data:
|
if chunk.data:
|
||||||
self._logger.debug(
|
self._logger.debug(f"Uploading chunk to vessel {self.vessel.name}")
|
||||||
f"Uploading chunk to vessel {self.vessel.name}")
|
|
||||||
self.vessel.pushChunk(chunk)
|
self.vessel.pushChunk(chunk)
|
||||||
else:
|
else:
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"No more data to upload to vessel {self.vessel.name} for file {fileobj.name} - compiling")
|
f"No more data to upload to vessel {self.vessel.name} for file {fileobj.name} - compiling"
|
||||||
|
)
|
||||||
self.vessel.compileComplete(remotefile)
|
self.vessel.compileComplete(remotefile)
|
||||||
|
|
||||||
def checkFileCompletion(self, fileobj: File) -> None:
|
def checkFileCompletion(self, fileobj: File) -> None:
|
||||||
db = self._dbclass()
|
db = self._dbclass()
|
||||||
complete = db.getCompletionByFileUUID(fileobj.uuid)
|
complete = db.getCompletionByFileUUID(fileobj.uuid)
|
||||||
del(db)
|
del db
|
||||||
|
|
||||||
for vessel in self._state["config"].vessels:
|
for vessel in self._state["config"].vessels:
|
||||||
if vessel.name not in complete and fileobj.directory.name not in vessel._ignoredirs:
|
if (
|
||||||
|
vessel.name not in complete
|
||||||
|
and fileobj.directory.name not in vessel._ignoredirs
|
||||||
|
):
|
||||||
return
|
return
|
||||||
|
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"File {fileobj.name} from Directory {fileobj.directory.name} transferred to all Vessels. Moving out of replication directory.")
|
f"File {fileobj.name} from Directory {fileobj.directory.name} transferred to all Vessels. Moving out of replication directory."
|
||||||
|
)
|
||||||
|
|
||||||
if fileobj.exists():
|
if fileobj.exists():
|
||||||
fileobj.moveCompleted()
|
fileobj.moveCompleted()
|
||||||
|
|
||||||
def processQueue(self) -> Optional[str]:
|
def processQueue(self) -> Optional[str]:
|
||||||
"""Return a file from the processing queue
|
"""Return a file from the processing queue"""
|
||||||
"""
|
|
||||||
self._logger.debug(
|
self._logger.debug(
|
||||||
f"Trying to fetch new file for vessel {self.vessel.name} from queue")
|
f"Trying to fetch new file for vessel {self.vessel.name} from queue"
|
||||||
|
)
|
||||||
for f in self._state["files"]:
|
for f in self._state["files"]:
|
||||||
if (not (f.uuid in self.vessel._uploaded)) and (not (f.directory.name in self.vessel._ignoredirs)):
|
if (not (f.uuid in self.vessel._uploaded)) and (
|
||||||
self._logger.debug(
|
not (f.directory.name in self.vessel._ignoredirs)
|
||||||
f"Using file {f.name} for vessel {self.vessel.name}")
|
):
|
||||||
|
self._logger.debug(f"Using file {f.name} for vessel {self.vessel.name}")
|
||||||
return f
|
return f
|
||||||
if f.uuid in self.vessel._uploaded:
|
if f.uuid in self.vessel._uploaded:
|
||||||
reason = "already uploaded"
|
reason = "already uploaded"
|
||||||
else:
|
else:
|
||||||
reason = "Directory ignored"
|
reason = "Directory ignored"
|
||||||
self._logger.debug(
|
self._logger.trace(
|
||||||
f"Disregarding file {f.name} for vessel {self.vessel.name} - {reason}")
|
f"Disregarding file {f.name} for vessel {self.vessel.name} - {reason}"
|
||||||
|
)
|
||||||
|
|
||||||
self._logger.debug(
|
self._logger.trace(f"Didn't find any new files for vessel {self.vessel.name}")
|
||||||
f"Didn't find any new files for vessel {self.vessel.name}")
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
# Constants for remote file status
|
# Constants for remote file status
|
||||||
|
|
||||||
STATUS_START = -1
|
STATUS_START = -1
|
||||||
STATUS_COMPLETE = -2
|
STATUS_COMPLETE = -2
|
||||||
|
|
|
@ -1,18 +1,47 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
from .classes.config import MonsterConfig
|
from contentmonster.classes.config import MonsterConfig
|
||||||
from .classes.vesselthread import VesselThread
|
from contentmonster.classes.vesselthread import VesselThread
|
||||||
from .classes.shorethread import ShoreThread
|
from contentmonster.classes.shorethread import ShoreThread
|
||||||
|
|
||||||
from multiprocessing import Manager
|
from multiprocessing import Manager
|
||||||
|
from argparse import ArgumentParser
|
||||||
|
|
||||||
import pathlib
|
import pathlib
|
||||||
import time
|
import time
|
||||||
|
import logging
|
||||||
|
import signal
|
||||||
|
|
||||||
if __name__ == '__main__':
|
# Setup basic logging
|
||||||
config_path = pathlib.Path(__file__).parent.absolute() / "settings.ini"
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
|
||||||
|
def setup_signal_handlers(shore, threads):
|
||||||
|
def signal_handler(signum, frame):
|
||||||
|
logging.info("Signal received - stopping threads")
|
||||||
|
shore.terminate()
|
||||||
|
for thread in threads:
|
||||||
|
thread.terminate()
|
||||||
|
exit(0)
|
||||||
|
signal.signal(signal.SIGINT, signal_handler)
|
||||||
|
signal.signal(signal.SIGTERM, signal_handler)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = ArgumentParser(description="ContentMonster Worker")
|
||||||
|
parser.add_argument("-c", "--config", help="Path to configuration file")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.config:
|
||||||
|
config_path = args.config
|
||||||
|
else:
|
||||||
|
config_path = "settings.ini"
|
||||||
|
|
||||||
config = MonsterConfig()
|
config = MonsterConfig()
|
||||||
config.readFile(config_path)
|
|
||||||
|
try:
|
||||||
|
config.readFile(config_path)
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Failed to read configuration: {e}")
|
||||||
|
return
|
||||||
|
|
||||||
with Manager() as manager:
|
with Manager() as manager:
|
||||||
state = manager.dict()
|
state = manager.dict()
|
||||||
|
@ -29,12 +58,14 @@ if __name__ == '__main__':
|
||||||
shore = ShoreThread(state)
|
shore = ShoreThread(state)
|
||||||
shore.start()
|
shore.start()
|
||||||
|
|
||||||
|
setup_signal_handlers(shore, threads)
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
except KeyboardInterrupt:
|
except Exception as e:
|
||||||
print("Keyboard interrupt received - stopping threads")
|
logging.error(f"Unexpected error: {e}")
|
||||||
shore.terminate()
|
break
|
||||||
for thread in threads:
|
|
||||||
thread.terminate()
|
if __name__ == '__main__':
|
||||||
exit()
|
main()
|
Loading…
Reference in a new issue