KumiSystems/contentmonster
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Add option to ignore a directory on a vessel

Browse source
This commit is contained in:
Kumi 2021-11-30 15:44:50 +01:00
parent 4de807b37e
commit c072f15b16
3 changed files with 31 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
README.md
View file

@ -50,7 +50,7 @@ Note: Currently, the same Location value is used on both the shore and the vesse
### Vessel
You can configure as many vessels to replicate your files to as you want by adding multiple `Vessel` sections. All configured directories are replicated to all vessels. If you want to use an SSH key to authenticate on the vessels, make sure that it is picked up by the local SSH agent (i.e. you can login using the key when connecting with the `ssh` command).
You can configure as many vessels to replicate your files to as you want by adding multiple `Vessel` sections. All configured directories are replicated to all vessels by default, but you can use the IgnoreDirs directive to exclude a directory from a given vessel. If you want to use an SSH key to authenticate on the vessels, make sure that it is picked up by the local SSH agent (i.e. you can login using the key when connecting with the `ssh` command).
```ini
[Vessel samplevessel] # Each vessel needs a unique name - here: "samplevessel"
@ -60,10 +60,9 @@ Username = replication # Username to authenticate as on the vessel (default: sa
Password = verysecret # Password to use to authenticate on the vessel (default: none, use SSH key)
Passphrase = moresecret # Passphrase of the SSH key you use to authenticate (default: none, key has no passphrase)
Port = 22 # Port of the SSH server on the vessel (default: 22)
IgnoreDirs = sampledir, anotherdir # Names of directories *not* to replicate to this vessel, separated by commas
```
Note: All configured directories are replicated to all of the configured vessels. This may be configurable in a future version.
## Running
To run the application after creating the `settings.ini`, navigate to ContentMonster's base directory in a terminal and make sure you are in the right virtual environment:
@ -108,4 +107,4 @@ systemctl enable --now contentmonster
systemctl status contentmonster # Check that the service started properly
```
The service should now start automatically after every reboot. You can use commands like `systemctl status contentmonster` and `journalctl -xeu contentmonster` to keep an eye on the status of the service.
The service should now start automatically after every reboot. You can use commands like `systemctl status contentmonster` and `journalctl -xeu contentmonster` to keep an eye on the status of the service.

11
classes/vessel.py
View file

@ -49,10 +49,13 @@ class Vessel:
if "Port" in config.keys():
port = config["Port"]
if "IgnoreDirs" in config.keys():
ignoredirs = [d.strip() for d in config["IgnoreDirs"].split(",")]
if "Address" in config.keys():
return cls(config.name.split()[1], config["Address"], username,
password, passphrase, tempdir)
password, passphrase, tempdir, ignoredirs)
else:
raise ValueError("Definition for Vessel " +
config.name.split()[1] + " does not contain Address!")
@ -60,7 +63,8 @@ class Vessel:
def __init__(self, name: str, address: str, username: Optional[str] = None,
password: Optional[str] = None, passphrase: Optional[str] = None,
port: Optional[int] = None, timeout: Optional[int] = None,
tempdir: Optional[Union[str, pathlib.Path]] = None) -> None:
tempdir: Optional[Union[str, pathlib.Path]] = None,
ignoredirs: list[Optional[str]] = []) -> None:
"""Initialize new Vessel object
Args:
@ -79,6 +83,7 @@ class Vessel:
self.timeout = timeout or 10
self._connection = None
self._uploaded = self.getUploadedFromDB() # Files already uploaded
self._ignoredirs = ignoredirs # Directories not replicated to this vessel
@property
def connection(self) -> Connection:
@ -178,4 +183,4 @@ class Vessel:
IOError: Raised if the directory does not exist but cannot be
created.
"""
self.connection.assertTempDirectory()
self.connection.assertTempDirectory()

26
classes/vesselthread.py
View file

@ -45,9 +45,10 @@ class VesselThread(Process):
@retry()
def assertDirectories(self) -> None:
for directory in self._state["config"].directories:
print(
f"Making sure directory {directory.name} exists on Vessel {self.vessel.name}")
self.vessel.connection.assertDirectories(directory)
if not directory.name in self.vessel._ignoredirs:
print(
f"Making sure directory {directory.name} exists on Vessel {self.vessel.name}")
self.vessel.connection.assertDirectories(directory)
@retry()
def upload(self) -> None:
@ -72,7 +73,7 @@ class VesselThread(Process):
if not directory:
self._logger.debug(
f"Directory {dirname} not specified in config - deleting File from Vessel {self.name}")
f"Directory {dirname} not specified in config - deleting File from Vessel {self.vessel.name}")
self.vessel.clearTempDir()
return
@ -86,6 +87,17 @@ class VesselThread(Process):
else:
fileobj = current
if fileobj.directory.name in self.vessel._ignoredirs:
self._logger.debug(
f"Not replicating Directory {fileobj.directory.name} to Vessel {self.vessel.name} - marking complete")
db = Database()
db.logCompletion(fileobj, self.vessel)
del(db)
self.checkFileCompletion(fileobj, self.vessel)
return
remotefile = RemoteFile(fileobj, self.vessel,
self._state["config"].chunksize)
@ -146,10 +158,12 @@ class VesselThread(Process):
complete = db.getCompletionByFileUUID(fileobj.uuid)
del(db)
for vessel in [v.name for v in self._state["config"].vessels]:
if vessel not in complete:
for vessel in self._state["config"].vessels:
if vessel.name not in complete and fileobj.directory.name not in vessel._ignoredirs:
return
self._logger.debug(
f"File {fileobj.name} from Directory {fileobj.directory.name} transferred to all Vessels. Moving out of replication directory.")
fileobj.moveCompleted()
def processQueue(self) -> Optional[str]: