Merge pull request #1668 from pbiering/login-cache

add optional cache for login result and htpasswd + fixes

final  version will be updated to 3.4.0 next

.github/workflows/test.yml

@@ -6,10 +6,8 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12.3', '3.13.0', pypy-3.9]
+        python-version: ['3.9', '3.10', '3.11', '3.12.3', '3.13.0', pypy-3.9]
         exclude:
-          - os: windows-latest
-            python-version: pypy-3.8
           - os: windows-latest
             python-version: pypy-3.9
     runs-on: ${{ matrix.os }}

CHANGELOG.md

@@ -1,6 +1,14 @@
 # Changelog
 
 ## 3.3.4.dev
+* Add: option [auth] cache_logins/cache_successful_logins_expiry/cache_failed_logins for caching logins
+* Improve: [auth] log used hash method and result on debug for htpasswd authentication
+* Improve: [auth] htpasswd file now read and verified on start
+* Add: option [auth] htpasswd_cache to automatic re-read triggered on change (mtime or size) instead reading on each request
+* Improve: [auth] htpasswd: module 'bcrypt' is no longer mandatory in case digest method not used in file
+* Improve: [auth] successful/failed login logs now type and whether result was taken from cache
+* Improve: [auth] constant execution time for failed logins independent of external backend or by htpasswd used digest method
+* Drop: support for Python 3.8
 
 ## 3.3.3
 * Add: display mtime_ns precision of storage folder with condition warning if too less

DOCUMENTATION.md

@@ -55,7 +55,7 @@ Follow one of the chapters below depending on your operating system.
 
 #### Linux / \*BSD
 
-First, make sure that **python** 3.8 or later and **pip** are installed. On most distributions it should be
+First, make sure that **python** 3.9 or later and **pip** are installed. On most distributions it should be
 enough to install the package ``python3-pip``.
 
 Then open a console and type:
@@ -812,6 +812,25 @@ Available backends:
 
 Default: `none`
 
+##### cache_logins
+
+Cache successful/failed logins until expiration time. Enable this to avoid
+overload of authentication backends.
+
+Default: `false`
+
+##### cache_successful_logins_expiry
+
+Expiration time of caching successful logins in seconds
+
+Default: `15`
+
+##### cache_failed_logins_expiry
+
+Expiration time of caching failed logins in seconds
+
+Default: `90`
+
 ##### htpasswd_filename
 
 Path to the htpasswd file.
@@ -853,6 +872,12 @@ Available methods:
 
 Default: `autodetect`
 
+##### htpasswd_cache
+
+Enable caching of htpasswd file based on size and mtime_ns
+
+Default: `False`
+
 ##### delay
 
 Average delay after failed login attempts in seconds.

config

@@ -62,6 +62,15 @@
 # Value: none | htpasswd | remote_user | http_x_remote_user | ldap | denyall
 #type = none
 
+# Cache logins for until expiration time
+#cache_logins = false
+
+# Expiration time for caching successful logins in seconds
+#cache_successful_logins_expiry = 15
+
+## Expiration time of caching failed logins in seconds
+#cache_failed_logins_expiry = 90
+
 # URI to the LDAP server
 #ldap_uri = ldap://localhost
 
@@ -103,6 +112,9 @@
 # bcrypt requires the installation of 'bcrypt' module.
 #htpasswd_encryption = autodetect
 
+# Enable caching of htpasswd file based on size and mtime_ns
+#htpasswd_cache = False
+
 # Incorrect authentication delay (seconds)
 #delay = 1
 

pyproject.toml

@@ -17,7 +17,6 @@ classifiers = [
     "License :: OSI Approved :: GNU General Public License (GPL)",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
@@ -28,7 +27,7 @@ classifiers = [
     "Topic :: Office/Business :: Groupware",
 ]
 urls = {Homepage = "https://radicale.org/"}
-requires-python = ">=3.8.0"
+requires-python = ">=3.9.0"
 dependencies = [
     "defusedxml",
     "passlib",

radicale/app/__init__.py

@@ -3,7 +3,7 @@
 # Copyright © 2008 Pascal Halter
 # Copyright © 2008-2017 Guillaume Ayoub
 # Copyright © 2017-2019 Unrud <unrud@outlook.com>
-# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
+# Copyright © 2024-2025 Peter Bieringer <pb@bieringer.de>
 #
 # This library is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -252,7 +252,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead,
                 self.configuration, environ, base64.b64decode(
                     authorization.encode("ascii"))).split(":", 1)
 
-        user = self._auth.login(login, password) or "" if login else ""
+        (user, info) = self._auth.login(login, password) or ("", "") if login else ("", "")
         if self.configuration.get("auth", "type") == "ldap":
             try:
                 logger.debug("Groups %r", ",".join(self._auth._ldap_groups))
@@ -260,16 +260,16 @@ class Application(ApplicationPartDelete, ApplicationPartHead,
             except AttributeError:
                 pass
         if user and login == user:
-            logger.info("Successful login: %r", user)
+            logger.info("Successful login: %r (%s)", user, info)
         elif user:
-            logger.info("Successful login: %r -> %r", login, user)
+            logger.info("Successful login: %r -> %r (%s)", login, user, info)
         elif login:
-            logger.warning("Failed login attempt from %s: %r",
-                           remote_host, login)
+            logger.warning("Failed login attempt from %s: %r (%s)",
+                           remote_host, login, info)
             # Random delay to avoid timing oracles and bruteforce attacks
             if self._auth_delay > 0:
                 random_delay = self._auth_delay * (0.5 + random.random())
-                logger.debug("Sleeping %.3f seconds", random_delay)
+                logger.debug("Failed login, sleeping random: %.3f sec", random_delay)
                 time.sleep(random_delay)
 
         if user and not pathutils.is_safe_path_component(user):

radicale/auth/__init__.py

@@ -3,7 +3,7 @@
 # Copyright © 2008 Pascal Halter
 # Copyright © 2008-2017 Guillaume Ayoub
 # Copyright © 2017-2022 Unrud <unrud@outlook.com>
-# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
+# Copyright © 2024-2025 Peter Bieringer <pb@bieringer.de>
 #
 # This library is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -29,6 +29,9 @@ Take a look at the class ``BaseAuth`` if you want to implement your own.
 
 """
 
+import hashlib
+import threading
+import time
 from typing import Sequence, Set, Tuple, Union, final
 
 from radicale import config, types, utils
@@ -57,6 +60,16 @@ class BaseAuth:
     _lc_username: bool
     _uc_username: bool
     _strip_domain: bool
+    _auth_delay: float
+    _failed_auth_delay: float
+    _type: str
+    _cache_logins: bool
+    _cache_successful: dict                 # login -> (digest, time_ns)
+    _cache_successful_logins_expiry: int
+    _cache_failed: dict                     # digest_failed -> (time_ns, login)
+    _cache_failed_logins_expiry: int
+    _cache_failed_logins_salt_ns: int       # persistent over runtime
+    _lock: threading.Lock
 
     def __init__(self, configuration: "config.Configuration") -> None:
         """Initialize BaseAuth.
@@ -75,6 +88,38 @@ class BaseAuth:
         logger.info("auth.uc_username: %s", self._uc_username)
         if self._lc_username is True and self._uc_username is True:
             raise RuntimeError("auth.lc_username and auth.uc_username cannot be enabled together")
+        self._auth_delay = configuration.get("auth", "delay")
+        logger.info("auth.delay: %f", self._auth_delay)
+        self._failed_auth_delay = 0
+        self._lock = threading.Lock()
+        # cache_successful_logins
+        self._cache_logins = configuration.get("auth", "cache_logins")
+        self._type = configuration.get("auth", "type")
+        if (self._type in ["dovecot", "ldap", "htpasswd"]) or (self._cache_logins is False):
+            logger.info("auth.cache_logins: %s", self._cache_logins)
+        else:
+            logger.info("auth.cache_logins: %s (but not required for type '%s' and disabled therefore)", self._cache_logins, self._type)
+            self._cache_logins = False
+        if self._cache_logins is True:
+            self._cache_successful_logins_expiry = configuration.get("auth", "cache_successful_logins_expiry")
+            if self._cache_successful_logins_expiry < 0:
+                raise RuntimeError("self._cache_successful_logins_expiry cannot be < 0")
+            self._cache_failed_logins_expiry = configuration.get("auth", "cache_failed_logins_expiry")
+            if self._cache_failed_logins_expiry < 0:
+                raise RuntimeError("self._cache_failed_logins_expiry cannot be < 0")
+            logger.info("auth.cache_successful_logins_expiry: %s seconds", self._cache_successful_logins_expiry)
+            logger.info("auth.cache_failed_logins_expiry: %s seconds", self._cache_failed_logins_expiry)
+            # cache init
+            self._cache_successful = dict()
+            self._cache_failed = dict()
+            self._cache_failed_logins_salt_ns = time.time_ns()
+
+    def _cache_digest(self, login: str, password: str, salt: str) -> str:
+        h = hashlib.sha3_512()
+        h.update(salt.encode())
+        h.update(login.encode())
+        h.update(password.encode())
+        return str(h.digest())
 
     def get_external_login(self, environ: types.WSGIEnviron) -> Union[
             Tuple[()], Tuple[str, str]]:
@@ -102,12 +147,132 @@ class BaseAuth:
 
         raise NotImplementedError
 
+    def _sleep_for_constant_exec_time(self, time_ns_begin: int):
+        """Sleep some time to reach a constant execution time for failed logins
+
+        Independent of time required by external backend or used digest methods
+
+        Increase final execution time in case initial limit exceeded
+
+        See also issue 591
+
+        """
+        time_delta = (time.time_ns() - time_ns_begin) / 1000 / 1000 / 1000
+        with self._lock:
+            # avoid that another thread is changing global value at the same time
+            failed_auth_delay = self._failed_auth_delay
+            failed_auth_delay_old = failed_auth_delay
+            if time_delta > failed_auth_delay:
+                # set new
+                failed_auth_delay = time_delta
+                # store globally
+                self._failed_auth_delay = failed_auth_delay
+        if (failed_auth_delay_old != failed_auth_delay):
+            logger.debug("Failed login constant execution time need increase of failed_auth_delay: %.9f -> %.9f sec", failed_auth_delay_old, failed_auth_delay)
+            # sleep == 0
+        else:
+            sleep = failed_auth_delay - time_delta
+            logger.debug("Failed login constant exection time alignment, sleeping: %.9f sec", sleep)
+            time.sleep(sleep)
+
     @final
-    def login(self, login: str, password: str) -> str:
+    def login(self, login: str, password: str) -> Tuple[str, str]:
+        time_ns_begin = time.time_ns()
+        result_from_cache = False
         if self._lc_username:
             login = login.lower()
         if self._uc_username:
             login = login.upper()
         if self._strip_domain:
             login = login.split('@')[0]
-        return self._login(login, password)
+        if self._cache_logins is True:
+            # time_ns is also used as salt
+            result = ""
+            digest = ""
+            time_ns = time.time_ns()
+            # cleanup failed login cache to avoid out-of-memory
+            cache_failed_entries = len(self._cache_failed)
+            if cache_failed_entries > 0:
+                logger.debug("Login failed cache investigation start (entries: %d)", cache_failed_entries)
+                self._lock.acquire()
+                cache_failed_cleanup = dict()
+                for digest in self._cache_failed:
+                    (time_ns_cache, login_cache) = self._cache_failed[digest]
+                    age_failed = int((time_ns - time_ns_cache) / 1000 / 1000 / 1000)
+                    if age_failed > self._cache_failed_logins_expiry:
+                        cache_failed_cleanup[digest] = (login_cache, age_failed)
+                cache_failed_cleanup_entries = len(cache_failed_cleanup)
+                logger.debug("Login failed cache cleanup start (entries: %d)", cache_failed_cleanup_entries)
+                if cache_failed_cleanup_entries > 0:
+                    for digest in cache_failed_cleanup:
+                        (login, age_failed) = cache_failed_cleanup[digest]
+                        logger.debug("Login failed cache entry for user+password expired: '%s' (age: %d > %d sec)", login_cache, age_failed, self._cache_failed_logins_expiry)
+                        del self._cache_failed[digest]
+                self._lock.release()
+                logger.debug("Login failed cache investigation finished")
+            # check for cache failed login
+            digest_failed = login + ":" + self._cache_digest(login, password, str(self._cache_failed_logins_salt_ns))
+            if self._cache_failed.get(digest_failed):
+                # login+password found in cache "failed" -> shortcut return
+                (time_ns_cache, login_cache) = self._cache_failed[digest]
+                age_failed = int((time_ns - time_ns_cache) / 1000 / 1000 / 1000)
+                logger.debug("Login failed cache entry for user+password found: '%s' (age: %d sec)", login_cache, age_failed)
+                self._sleep_for_constant_exec_time(time_ns_begin)
+                return ("", self._type + " / cached")
+            if self._cache_successful.get(login):
+                # login found in cache "successful"
+                (digest_cache, time_ns_cache) = self._cache_successful[login]
+                digest = self._cache_digest(login, password, str(time_ns_cache))
+                if digest == digest_cache:
+                    age_success = int((time_ns - time_ns_cache) / 1000 / 1000 / 1000)
+                    if age_success > self._cache_successful_logins_expiry:
+                        logger.debug("Login successful cache entry for user+password found but expired: '%s' (age: %d > %d sec)", login, age_success, self._cache_successful_logins_expiry)
+                        # delete expired success from cache
+                        del self._cache_successful[login]
+                        digest = ""
+                    else:
+                        logger.debug("Login successful cache entry for user+password found: '%s' (age: %d sec)", login, age_success)
+                        result = login
+                        result_from_cache = True
+                else:
+                    logger.debug("Login successful cache entry for user+password not matching: '%s'", login)
+            else:
+                # login not found in cache, caculate always to avoid timing attacks
+                digest = self._cache_digest(login, password, str(time_ns))
+            if result == "":
+                # verify login+password via configured backend
+                logger.debug("Login verification for user+password via backend: '%s'", login)
+                result = self._login(login, password)
+                if result != "":
+                    logger.debug("Login successful for user+password via backend: '%s'", login)
+                    if digest == "":
+                        # successful login, but expired, digest must be recalculated
+                        digest = self._cache_digest(login, password, str(time_ns))
+                    # store successful login in cache
+                    self._lock.acquire()
+                    self._cache_successful[login] = (digest, time_ns)
+                    self._lock.release()
+                    logger.debug("Login successful cache for user set: '%s'", login)
+                    if self._cache_failed.get(digest_failed):
+                        logger.debug("Login failed cache for user cleared: '%s'", login)
+                        del self._cache_failed[digest_failed]
+                else:
+                    logger.debug("Login failed for user+password via backend: '%s'", login)
+                    self._lock.acquire()
+                    self._cache_failed[digest_failed] = (time_ns, login)
+                    self._lock.release()
+                    logger.debug("Login failed cache for user set: '%s'", login)
+            if result_from_cache is True:
+                if result == "":
+                    self._sleep_for_constant_exec_time(time_ns_begin)
+                return (result, self._type + " / cached")
+            else:
+                if result == "":
+                    self._sleep_for_constant_exec_time(time_ns_begin)
+                return (result, self._type)
+        else:
+            # self._cache_logins is False
+            result = self._login(login, password)
+            if result == "":
+                self._sleep_for_constant_exec_time(time_ns_begin)
+            return (result, self._type)

radicale/auth/htpasswd.py

@@ -3,7 +3,7 @@
 # Copyright © 2008 Pascal Halter
 # Copyright © 2008-2017 Guillaume Ayoub
 # Copyright © 2017-2019 Unrud <unrud@outlook.com>
-# Copyright © 2024 Peter Bieringer <pb@bieringer.de>
+# Copyright © 2024-2025 Peter Bieringer <pb@bieringer.de>
 #
 # This library is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -50,7 +50,10 @@ When bcrypt is installed:
 
 import functools
 import hmac
-from typing import Any
+import os
+import threading
+import time
+from typing import Any, Tuple
 
 from passlib.hash import apr_md5_crypt, sha256_crypt, sha512_crypt
 
@@ -61,15 +64,34 @@ class Auth(auth.BaseAuth):
 
     _filename: str
     _encoding: str
+    _htpasswd: dict             # login -> digest
+    _htpasswd_mtime_ns: int
+    _htpasswd_size: int
+    _htpasswd_ok: bool
+    _htpasswd_not_ok_time: float
+    _htpasswd_not_ok_reminder_seconds: int
+    _htpasswd_bcrypt_use: int
+    _htpasswd_cache: bool
+    _has_bcrypt: bool
+    _lock: threading.Lock
 
     def __init__(self, configuration: config.Configuration) -> None:
         super().__init__(configuration)
         self._filename = configuration.get("auth", "htpasswd_filename")
+        logger.info("auth htpasswd file: %r", self._filename)
         self._encoding = configuration.get("encoding", "stock")
+        logger.info("auth htpasswd file encoding: %r", self._encoding)
+        self._htpasswd_cache = configuration.get("auth", "htpasswd_cache")
+        logger.info("auth htpasswd cache: %s", self._htpasswd_cache)
         encryption: str = configuration.get("auth", "htpasswd_encryption")
-
         logger.info("auth htpasswd encryption is 'radicale.auth.htpasswd_encryption.%s'", encryption)
 
+        self._has_bcrypt = False
+        self._htpasswd_ok = False
+        self._htpasswd_not_ok_reminder_seconds = 60 # currently hardcoded
+        (self._htpasswd_ok, self._htpasswd_bcrypt_use, self._htpasswd, self._htpasswd_size, self._htpasswd_mtime_ns) = self._read_htpasswd(True, False)
+        self._lock = threading.Lock()
+
         if encryption == "plain":
             self._verify = self._plain
         elif encryption == "md5":
@@ -82,35 +104,45 @@ class Auth(auth.BaseAuth):
             try:
                 import bcrypt
             except ImportError as e:
-                raise RuntimeError(
-                    "The htpasswd encryption method 'bcrypt' or 'autodetect' requires "
-                    "the bcrypt module.") from e
+                if (encryption == "autodetect") and (self._htpasswd_bcrypt_use == 0):
+                    logger.warning("auth htpasswd encryption is 'radicale.auth.htpasswd_encryption.%s' which can require bycrypt module, but currently no entries found", encryption)
+                else:
+                    raise RuntimeError(
+                        "The htpasswd encryption method 'bcrypt' or 'autodetect' requires "
+                        "the bcrypt module (entries found: %d)." % self._htpasswd_bcrypt_use) from e
+            else:
+                if encryption == "autodetect":
+                    if self._htpasswd_bcrypt_use == 0:
+                        logger.info("auth htpasswd encryption is 'radicale.auth.htpasswd_encryption.%s' and bycrypt module found, but currently not required", encryption)
+                    else:
+                        logger.info("auth htpasswd encryption is 'radicale.auth.htpasswd_encryption.%s' and bycrypt module found (bcrypt entries found: %d)", encryption, self._htpasswd_bcrypt_use)
             if encryption == "bcrypt":
                 self._verify = functools.partial(self._bcrypt, bcrypt)
             else:
                 self._verify = self._autodetect
                 self._verify_bcrypt = functools.partial(self._bcrypt, bcrypt)
+            self._has_bcrypt = True
         else:
             raise RuntimeError("The htpasswd encryption method %r is not "
                                "supported." % encryption)
 
-    def _plain(self, hash_value: str, password: str) -> bool:
+    def _plain(self, hash_value: str, password: str) -> tuple[str, bool]:
         """Check if ``hash_value`` and ``password`` match, plain method."""
-        return hmac.compare_digest(hash_value.encode(), password.encode())
+        return ("PLAIN", hmac.compare_digest(hash_value.encode(), password.encode()))
 
-    def _bcrypt(self, bcrypt: Any, hash_value: str, password: str) -> bool:
-        return bcrypt.checkpw(password=password.encode('utf-8'), hashed_password=hash_value.encode())
+    def _bcrypt(self, bcrypt: Any, hash_value: str, password: str) -> tuple[str, bool]:
+        return ("BCRYPT", bcrypt.checkpw(password=password.encode('utf-8'), hashed_password=hash_value.encode()))
 
-    def _md5apr1(self, hash_value: str, password: str) -> bool:
-        return apr_md5_crypt.verify(password, hash_value.strip())
+    def _md5apr1(self, hash_value: str, password: str) -> tuple[str, bool]:
+        return ("MD5-APR1", apr_md5_crypt.verify(password, hash_value.strip()))
 
-    def _sha256(self, hash_value: str, password: str) -> bool:
-        return sha256_crypt.verify(password, hash_value.strip())
+    def _sha256(self, hash_value: str, password: str) -> tuple[str, bool]:
+        return ("SHA-256", sha256_crypt.verify(password, hash_value.strip()))
 
-    def _sha512(self, hash_value: str, password: str) -> bool:
-        return sha512_crypt.verify(password, hash_value.strip())
+    def _sha512(self, hash_value: str, password: str) -> tuple[str, bool]:
+        return ("SHA-512", sha512_crypt.verify(password, hash_value.strip()))
 
-    def _autodetect(self, hash_value: str, password: str) -> bool:
+    def _autodetect(self, hash_value: str, password: str) -> tuple[str, bool]:
         if hash_value.startswith("$apr1$", 0, 6) and len(hash_value) == 37:
             # MD5-APR1
             return self._md5apr1(hash_value, password)
@@ -127,6 +159,89 @@ class Auth(auth.BaseAuth):
             # assumed plaintext
             return self._plain(hash_value, password)
 
+    def _read_htpasswd(self, init: bool, suppress: bool) -> Tuple[bool, int, dict, int, int]:
+        """Read htpasswd file
+
+        init == True: stop on error
+        init == False: warn/skip on error and set mark to log reminder every interval
+        suppress == True: suppress warnings, change info to debug (used in non-caching mode)
+        suppress == False: do not suppress warnings (used in caching mode)
+
+        """
+        htpasswd_ok = True
+        bcrypt_use = 0
+        if (init is True) or (suppress is True):
+            info = "Read"
+        else:
+            info = "Re-read"
+        if suppress is False:
+            logger.info("%s content of htpasswd file start: %r", info, self._filename)
+        else:
+            logger.debug("%s content of htpasswd file start: %r", info, self._filename)
+        htpasswd: dict[str, str] = dict()
+        entries = 0
+        duplicates = 0
+        errors = 0
+        try:
+            with open(self._filename, encoding=self._encoding) as f:
+                line_num = 0
+                for line in f:
+                    line_num += 1
+                    line = line.rstrip("\n")
+                    if line.lstrip() and not line.lstrip().startswith("#"):
+                        try:
+                            login, digest = line.split(":", maxsplit=1)
+                            skip = False
+                            if login == "" or digest == "":
+                                if init is True:
+                                    raise ValueError("htpasswd file contains problematic line not matching <login>:<digest> in line: %d" % line_num)
+                                else:
+                                    errors += 1
+                                    logger.warning("htpasswd file contains problematic line not matching <login>:<digest> in line: %d (ignored)", line_num)
+                                    htpasswd_ok = False
+                                    skip = True
+                            else:
+                                if htpasswd.get(login):
+                                    duplicates += 1
+                                    if init is True:
+                                        raise ValueError("htpasswd file contains duplicate login: '%s'", login, line_num)
+                                    else:
+                                        logger.warning("htpasswd file contains duplicate login: '%s' (line: %d / ignored)", login, line_num)
+                                        htpasswd_ok = False
+                                        skip = True
+                                else:
+                                    if digest.startswith("$2y$", 0, 4) and len(digest) == 60:
+                                        if init is True:
+                                            bcrypt_use += 1
+                                        else:
+                                            if self._has_bcrypt is False:
+                                                logger.warning("htpasswd file contains bcrypt digest login: '%s' (line: %d / ignored because module is not loaded)", login, line_num)
+                                                skip = True
+                                                htpasswd_ok = False
+                            if skip is False:
+                                htpasswd[login] = digest
+                                entries += 1
+                        except ValueError as e:
+                            if init is True:
+                                raise RuntimeError("Invalid htpasswd file %r: %s" % (self._filename, e)) from e
+        except OSError as e:
+            if init is True:
+                raise RuntimeError("Failed to load htpasswd file %r: %s" % (self._filename, e)) from e
+            else:
+                logger.warning("Failed to load htpasswd file on re-read: %r" % self._filename)
+                htpasswd_ok = False
+        htpasswd_size = os.stat(self._filename).st_size
+        htpasswd_mtime_ns = os.stat(self._filename).st_mtime_ns
+        if suppress is False:
+            logger.info("%s content of htpasswd file done: %r (entries: %d, duplicates: %d, errors: %d)", info, self._filename, entries, duplicates, errors)
+        else:
+            logger.debug("%s content of htpasswd file done: %r (entries: %d, duplicates: %d, errors: %d)", info, self._filename, entries, duplicates, errors)
+        if htpasswd_ok is True:
+            self._htpasswd_not_ok_time = 0
+        else:
+            self._htpasswd_not_ok_time = time.time()
+        return (htpasswd_ok, bcrypt_use, htpasswd, htpasswd_size, htpasswd_mtime_ns)
+
     def _login(self, login: str, password: str) -> str:
         """Validate credentials.
 
@@ -134,30 +249,48 @@ class Auth(auth.BaseAuth):
         hash (encrypted password) and check hash against password,
         using the method specified in the Radicale config.
 
-        The content of the file is not cached because reading is generally a
-        very cheap operation, and it's useful to get live updates of the
-        htpasswd file.
+        Optional: the content of the file is cached and live updates will be detected by
+        comparing mtime_ns and size
 
         """
-        try:
-            with open(self._filename, encoding=self._encoding) as f:
-                for line in f:
-                    line = line.rstrip("\n")
-                    if line.lstrip() and not line.lstrip().startswith("#"):
-                        try:
-                            hash_login, hash_value = line.split(
-                                ":", maxsplit=1)
-                            # Always compare both login and password to avoid
-                            # timing attacks, see #591.
-                            login_ok = hmac.compare_digest(
-                                hash_login.encode(), login.encode())
-                            password_ok = self._verify(hash_value, password)
-                            if login_ok and password_ok:
-                                return login
-                        except ValueError as e:
-                            raise RuntimeError("Invalid htpasswd file %r: %s" %
-                                               (self._filename, e)) from e
-        except OSError as e:
-            raise RuntimeError("Failed to load htpasswd file %r: %s" %
-                               (self._filename, e)) from e
+        login_ok = False
+        digest: str
+        if self._htpasswd_cache is True:
+            # check and re-read file if required
+            with self._lock:
+                htpasswd_size = os.stat(self._filename).st_size
+                htpasswd_mtime_ns = os.stat(self._filename).st_mtime_ns
+                if (htpasswd_size != self._htpasswd_size) or (htpasswd_mtime_ns != self._htpasswd_mtime_ns):
+                    (self._htpasswd_ok, self._htpasswd_bcrypt_use, self._htpasswd, self._htpasswd_size, self._htpasswd_mtime_ns) = self._read_htpasswd(False, False)
+                    self._htpasswd_not_ok_time = 0
+
+            # log reminder of problemantic file every interval
+            current_time = time.time()
+            if (self._htpasswd_ok is False):
+                if (self._htpasswd_not_ok_time > 0):
+                    if (current_time - self._htpasswd_not_ok_time) > self._htpasswd_not_ok_reminder_seconds:
+                        logger.warning("htpasswd file still contains issues (REMINDER, check warnings in the past): %r" % self._filename)
+                        self._htpasswd_not_ok_time = current_time
+                else:
+                    self._htpasswd_not_ok_time = current_time
+
+            if self._htpasswd.get(login):
+                digest = self._htpasswd[login]
+                login_ok = True
+        else:
+            # read file on every request
+            (htpasswd_ok, htpasswd_bcrypt_use, htpasswd, htpasswd_size, htpasswd_mtime_ns) = self._read_htpasswd(False, True)
+            if htpasswd.get(login):
+                digest = htpasswd[login]
+                login_ok = True
+
+        if login_ok is True:
+            (method, password_ok) = self._verify(digest, password)
+            logger.debug("Login verification successful for user: '%s' (method '%s')", login, method)
+            if password_ok:
+                return login
+            else:
+                logger.debug("Login verification failed for user: '%s' ( method '%s')", login, method)
+        else:
+            logger.debug("Login verification user not found: '%s'", login)
         return ""

radicale/config.py

@@ -183,6 +183,18 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([
             "help": "authentication method",
             "type": str_or_callable,
             "internal": auth.INTERNAL_TYPES}),
+        ("cache_logins", {
+            "value": "false",
+            "help": "cache successful/failed logins for until expiration time",
+            "type": bool}),
+        ("cache_successful_logins_expiry", {
+            "value": "15",
+            "help": "expiration time for caching successful logins in seconds",
+            "type": int}),
+        ("cache_failed_logins_expiry", {
+            "value": "90",
+            "help": "expiration time for caching failed logins in seconds",
+            "type": int}),
         ("htpasswd_filename", {
             "value": "/etc/radicale/users",
             "help": "htpasswd filename",
@@ -191,6 +203,10 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([
             "value": "autodetect",
             "help": "htpasswd encryption method",
             "type": str}),
+        ("htpasswd_cache", {
+            "value": "False",
+            "help": "enable caching of htpasswd file",
+            "type": bool}),
         ("dovecot_socket", {
             "value": "/var/run/dovecot/auth-client",
             "help": "dovecot auth socket",

radicale/tests/test_auth.py

@@ -2,7 +2,7 @@
 # Copyright © 2012-2016 Jean-Marc Martins
 # Copyright © 2012-2017 Guillaume Ayoub
 # Copyright © 2017-2022 Unrud <unrud@outlook.com>
-# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
+# Copyright © 2024-2025 Peter Bieringer <pb@bieringer.de>
 #
 # This library is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,6 +23,7 @@ Radicale tests with simple requests and authentication.
 """
 
 import base64
+import logging
 import os
 import sys
 from typing import Iterable, Tuple, Union
@@ -101,6 +102,51 @@ class TestBaseAuthRequests(BaseTest):
     def test_htpasswd_multi(self) -> None:
         self._test_htpasswd("plain", "ign:ign\ntmp:bepo")
 
+    # login cache successful
+    def test_htpasswd_login_cache_successful_plain(self, caplog) -> None:
+        caplog.set_level(logging.INFO)
+        self.configure({"auth": {"cache_logins": "True"}})
+        self._test_htpasswd("plain", "tmp:bepo", (("tmp", "bepo", True), ("tmp", "bepo", True)))
+        htpasswd_found = False
+        htpasswd_cached_found = False
+        for line in caplog.messages:
+            if line == "Successful login: 'tmp' (htpasswd)":
+                htpasswd_found = True
+            elif line == "Successful login: 'tmp' (htpasswd / cached)":
+                htpasswd_cached_found = True
+        if (htpasswd_found is False) or (htpasswd_cached_found is False):
+            raise ValueError("Logging misses expected log lines")
+
+    # login cache failed
+    def test_htpasswd_login_cache_failed_plain(self, caplog) -> None:
+        caplog.set_level(logging.INFO)
+        self.configure({"auth": {"cache_logins": "True"}})
+        self._test_htpasswd("plain", "tmp:bepo", (("tmp", "bepo1", False), ("tmp", "bepo1", False)))
+        htpasswd_found = False
+        htpasswd_cached_found = False
+        for line in caplog.messages:
+            if line == "Failed login attempt from unknown: 'tmp' (htpasswd)":
+                htpasswd_found = True
+            elif line == "Failed login attempt from unknown: 'tmp' (htpasswd / cached)":
+                htpasswd_cached_found = True
+        if (htpasswd_found is False) or (htpasswd_cached_found is False):
+            raise ValueError("Logging misses expected log lines")
+
+    # htpasswd file cache
+    def test_htpasswd_file_cache(self, caplog) -> None:
+        self.configure({"auth": {"htpasswd_cache": "True"}})
+        self._test_htpasswd("plain", "tmp:bepo")
+
+    # detection of broken htpasswd file entries
+    def test_htpasswd_broken(self) -> None:
+        for userpass in ["tmp:", ":tmp"]:
+            try:
+                self._test_htpasswd("plain", userpass)
+            except RuntimeError:
+                pass
+            else:
+                raise
+
     @pytest.mark.skipif(sys.platform == "win32", reason="leading and trailing "
                         "whitespaces not allowed in file names")
     def test_htpasswd_whitespace_user(self) -> None:

setup.py.legacy

@@ -61,7 +61,7 @@ setup(
     install_requires=install_requires,
     extras_require={"test": test_requires, "bcrypt": bcrypt_requires, "ldap": ldap_requires},
     keywords=["calendar", "addressbook", "CalDAV", "CardDAV"],
-    python_requires=">=3.8.0",
+    python_requires=">=3.9.0",
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
@@ -71,7 +71,6 @@ setup(
         "License :: OSI Approved :: GNU General Public License (GPL)",
         "Operating System :: OS Independent",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",