Some refactoring to get cron running

Moved dbsettings documentation to Gitlab wiki

celerybeat.pid

@@ -0,0 +1 @@
+7348

core/apps.py

@@ -1,5 +1,4 @@
 from django.apps import AppConfig
 
-
 class CoreConfig(AppConfig):
     name = 'core'

core/classes/cron.py

@@ -0,0 +1,38 @@
+from django.utils import timezone
+
+from core.models.cron import CronLog
+
+from dbsettings.functions import getValue
+
+from parse_crontab import CronTab
+
+class Cronjob:
+    def __init__(self, name, crondef, lock=getValue("core.cron.lock", 300)):
+        self.name = name
+        self.crondef = crondef
+        self.lock = lock
+    
+    @property
+    def is_running(self):
+        now = timezone.now()
+        maxage = now - timezone.timedelta(seconds=self.lock)
+        CronLog.objects.filter(task=self.name, locked=True, execution__lt=maxage).update(locked=False)
+        return True if CronLog.objects.filter(task=self.name, locked=True) else False
+
+    @property
+    def next_run(self):
+        runs = CronLog.objects.filter(task=self.name)
+        if not runs:
+            CronLog.objects.create(task=self.name, locked=False)
+            return self.next_run
+
+        lastrun = runs.latest("execution").execution
+        return lastrun + timezone.timedelta(seconds=CronTab(self.crondef).next(lastrun))
+
+    @property
+    def is_due(self):
+        return self.next_run <= timezone.now()
+    
+    def run(self):
+        from core.tasks.cron import run_cron
+        run_cron.delay(self.name)

core/cron.py

@@ -0,0 +1,22 @@
+from core.classes.cron import Cronjob
+from core.helpers.auth import clear_login_log
+
+CRONDEFINITIONS = []
+CRONFUNCTIONS = {}
+
+### Demonstration Cronjob
+
+def debug_job():
+    return "Test"
+
+debug_cron = Cronjob("core.debug_job", "* * * * *")
+
+# CRONFUNCTIONS["core.debug_job"] = debug_job
+# CRONDEFINITIONS.append(debug_cron)
+
+### Remove old entries from the login log
+
+loginlog_cron = Cronjob("core.clear_login_log", "* * * * *")
+
+CRONFUNCTIONS["core.clear_login_log"] = clear_login_log
+CRONDEFINITIONS.append(loginlog_cron)

core/helpers/auth.py

@@ -5,6 +5,7 @@ from core.helpers.request import get_client_ip
 
 from django.urls import reverse
 from django.contrib import messages
+from django.utils import timezone
 
 from dbsettings.functions import getValue
 
@@ -19,4 +20,8 @@ def login_fail(request, user=None, message=None):
         messages.error(request, message)
 
 def login_success(request, user):
-    LoginLog.objects.create(user=user, ip=get_client_ip(request), success=True)
+    LoginLog.objects.create(user=user, ip=get_client_ip(request), success=True)
+
+def clear_login_log(maxage=int(getValue("core.auth.ratelimit.period", 600))):
+    timestamp = timezone.now() - timezone.timedelta(seconds=maxage)
+    LoginLog.objects.filter(timestamp__lt=timestamp).delete()

core/helpers/cron.py

@@ -0,0 +1,13 @@
+from django_celery_beat.models import PeriodicTask, IntervalSchedule
+
+def setup_cron():
+    schedule, created = IntervalSchedule.objects.get_or_create(
+        every=1,
+        period=IntervalSchedule.MINUTES,
+    )
+
+    PeriodicTask.objects.get_or_create(
+        interval=schedule,
+        name='Expephacron',
+        task='cron',
+    )

core/helpers/mail.py

@@ -1,7 +1,5 @@
 from django.conf import settings
 
-from core.modules.mail import providers, templates
-from core.tasks.mail import send_mail as send_mail_task
 from core.exceptions.mail import NoSuchTemplate
 
 from dbsettings.functions import getValue
@@ -9,12 +7,14 @@ from dbsettings.functions import getValue
 import os.path
 
 def get_provider_by_name(name, fallback=True):
+    from core.modules.mail import providers
     return providers.get(name, None) or providers["smtp"]
 
 def get_default_provider(fallback=True):
     return get_provider_by_name(getValue("core.email.provider", "smtp"), fallback)
 
 def send_mail(provider=get_default_provider(), **kwargs):
+    from core.tasks.mail import send_mail as send_mail_task
     provider = get_provider_by_name(provider) if type(provider) == str else provider
     return send_mail_task.delay(provider, **kwargs)
 
@@ -22,6 +22,7 @@ def simple_send_mail(subject, content, recipients, cc=[], bcc=[], headers={}):
     return send_mail(subject=subject, content=content, recipients=recipients, cc=cc, bcc=bcc, headers=headers)
 
 def get_template(template_name, format="txt", **kwargs):
+    from core.modules.mail import templates
     try:
         template = templates[template_name][format]
     except KeyError:

core/management/commands/setupcron.py

@@ -0,0 +1,9 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from core.helpers.cron import setup_cron
+
+class Command(BaseCommand):
+    help = 'Enables the cron system'
+
+    def handle(self, *args, **options):
+        setup_cron()

core/models/cron.py

@@ -0,0 +1,6 @@
+from django.db.models import Model, CharField, DateTimeField, BooleanField
+
+class CronLog(Model):
+    task = CharField(max_length=255)
+    execution = DateTimeField(auto_now_add=True)
+    locked = BooleanField(default=True)

core/modules/cron.py

@@ -0,0 +1,18 @@
+import importlib
+
+from django.conf import settings
+
+cronfunctions = {}
+
+crondefinitions = []
+
+for module in ["core"] + settings.EXPEPHALON_MODULES:
+    try:
+        moc = importlib.import_module(f"{module}.cron")
+        for name, fun in moc.CRONFUNCTIONS.items():
+            if name in cronfunctions.keys():
+                raise ValueError(f"Error in {module}: Cron function with name {name} already registered!")
+            cronfunctions[name] = fun
+        crondefinitions += moc.CRONDEFINITIONS
+    except (AttributeError, ModuleNotFoundError):
+        continue

core/tasks/__init__.py

@@ -1 +1,2 @@
-from core.tasks.mail import *
+from core.tasks.mail import send_mail
+from core.tasks.cron import process_crons, run_cron

core/tasks/cron.py

@@ -0,0 +1,28 @@
+from celery import shared_task, task
+from celery.utils.log import get_task_logger
+
+logger = get_task_logger(__name__)
+
+@task(name="cron")
+def process_crons():
+    from core.modules.cron import crondefinitions
+
+    for definition in crondefinitions:
+        print(definition.next_run)
+        if definition.is_due and not definition.is_running:
+            definition.run()
+
+@shared_task
+def run_cron(name, *args, **kwargs):
+    from core.models.cron import CronLog
+    from core.modules.cron import cronfunctions
+
+    log = CronLog.objects.create(task=name)
+    try:
+        output = cronfunctions[name]()
+        if output:
+            logger.debug(f"[{name}] {output}")
+    except Exception as e:
+        logger.error(f"[{name}] {str(e)}")
+    log.locked = False
+    log.save()

dbsettings_keys.md

@@ -1,87 +0,0 @@
-This document includes all database config keys currently used by Expephalon itself.
-
-Third-party modules must not use keys in the core.\*, custom.\*, dbsettings.\* or expephalon.\* namespaces, as well as in the namespaces of official modules (chat.\*, demomodule.\*, kumisms.\*, playsms.\*, smsotp.\*, totp.\*), and should use namespaces which are specific enough to prevent collisions with other third-party modules or future official modules.
-
-The custom.\* is reserved for settings created manually by administrators for use in custom scripts or other reasons.
-
-[[_TOC_]]
-
-## Base configuration
-
-### core.base_url
-
-**Description**: URL under which this Expephalon installation is available, including protocol, generally without trailing slash, used for generation of absolute URLs
-
-**Default value:** http://localhost:8000
-
-### core.title
-
-**Description**: Title of the Expephalon installation, used in page titles and anywhere else the name of the site is referenced
-
-**Default value:** Expephalon
-
-## Authentication system
-
-### core.auth.otp.max_age
-
-**Description:** Maximum time from starting to finishing a one-time-password flow (in seconds)
-
-**Default value:** 300
-
-### core.auth.pwreset.max_age
-
-**Description:** Maximum time between creation and usage of a password reset token (in seconds)
-
-**Default value:** 86400
-
-### core.auth.ratelimit.attempts
-
-**Description:** Maximum number of invalid login attempts in last core.auth.ratelimit.period seconds from individual IP before blocking
-
-**Default value:** 5
-
-### core.auth.ratelimit.block
-
-**Description:** Period for which to block further login attempts from individual IP after c.a.r.attempts failures in c.a.r.period seconds (in seconds)
-
-**Default value:** 3600
-
-### core.auth.ratelimit.period
-
-**Description:** Period in which to check for previous failed login attempts for ratelimiting (in seconds)
-
-**Default value:** 600
-
-## Mail
-
-### core.mail.sender
-
-**Description:** Email address to be used as sender of outgoing mail
-
-**Default value:** "Expephalon" \<expephalon@localhost\>
-
-### core.smtp.host
-
-**Description:** Hostname of the SMTP server to be used for outgoing mail
-
-**Default value:** localhost
-
-### core.smtp.username
-
-**Description:** Username to authenticate to the SMTP server with
-
-**Default value:** (None)
-
-### core.smtp.password
-
-**Description:** Password to authenticate to the SMTP server with
-
-**Default value:** (None)
-
-## SMS
-
-### core.sms.default
-
-**Description:** Name of the default SMS provider to be used by Expephalon - must be the unique return value of the provider's get_name property
-
-**Default value:** (None, effectively disabling SMS - doesn't make sense without an SMS provider module installed)

expephalon/celery.py

@@ -2,20 +2,11 @@ import os
 
 from celery import Celery
 
-# set the default Django settings module for the 'celery' program.
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'expephalon.settings')
 
 app = Celery('expephalon')
 
-# Using a string here means the worker doesn't have to serialize
-# the configuration object to child processes.
-# - namespace='CELERY' means all celery-related configuration keys
-#   should have a `CELERY_` prefix.
 app.config_from_object('django.conf:settings', namespace='CELERY')
 
-# Load task modules from all registered Django app configs.
 app.autodiscover_tasks()
 
-@app.task(bind=True)
-def debug_task(self):
-    print('Request: {0!r}'.format(self.request))

requirements.txt

@@ -14,3 +14,4 @@ django-celery-beat
 python-memcached
 django-countries
 pyuca
+git+https://kumig.it/kumisystems/parse_crontab.git