Merge pull request #1642 from pbiering/storage-cache-separation

Storage cache separation

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 3.3.2.dev
+* Fix: debug logging in rights/from_file
+* Add: option [storage] use_cache_subfolder_for_item for storing item cache outside collection-root
+
 ## 3.3.1
 
 * Add: option [auth] type=dovecot

DOCUMENTATION.md

@@ -1005,6 +1005,12 @@ Folder for storing local collections, created if not present.
 
 Default: `/var/lib/radicale/collections`
 
+##### use_cache_subfolder_for_item
+
+Use subfolder `collections-cache' for cache file structure of item instead of inside collection folders, created if not present
+
+Default: `False`
+
 ##### max_sync_token_age
 
 Delete sync-token that are older than the specified time. (seconds)

config

@@ -138,6 +138,9 @@
 # Folder for storing local collections, created if not present
 #filesystem_folder = /var/lib/radicale/collections
 
+# Use subfolder `collections-cache' for item cache file structure instead of inside collection folder
+#use_cache_subfolder_for_item = False
+
 # Delete sync token that are older (seconds)
 #max_sync_token_age = 2592000
 

radicale/config.py

@@ -279,6 +279,10 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([
             "value": "/var/lib/radicale/collections",
             "help": "path where collections are stored",
             "type": filepath}),
+        ("use_cache_subfolder_for_item", {
+            "value": "False",
+            "help": "use subfolder `collections-cache' for item cache file structure instead of inside collection folder",
+            "type": bool}),
         ("max_sync_token_age", {
             "value": "2592000",  # 30 days
             "help": "delete sync token that are older",

radicale/rights/from_file.py

@@ -1,6 +1,7 @@
 # This file is part of Radicale - CalDAV and CardDAV server
 # Copyright © 2012-2017 Guillaume Ayoub
-# Copyright © 2017-2019 Unrud <unrud@outlook.com>
+# Copyright © 2017-2021 Unrud <unrud@outlook.com>
+# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
 #
 # This library is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -99,12 +100,9 @@ class Rights(rights.BaseRights):
                              user, sane_path, user_pattern,
                              collection_pattern, section, permission)
                 return permission
-            logger.debug("Rule %r:%r doesn't match %r:%r from section %r",
-                         user, sane_path, user_pattern, collection_pattern,
-                         section)
             if self._log_rights_rule_doesnt_match_on_debug:
                 logger.debug("Rule %r:%r doesn't match %r:%r from section %r",
                              user, sane_path, user_pattern, collection_pattern,
                              section)
-        logger.info("Rights: %r:%r doesn't match any section", user, sane_path)
+        logger.debug("Rights: %r:%r doesn't match any section", user, sane_path)
         return ""

radicale/storage/__init__.py

@@ -1,7 +1,7 @@
 # This file is part of Radicale - CalDAV and CardDAV server
 # Copyright © 2014 Jean-Marc Martins
 # Copyright © 2012-2017 Guillaume Ayoub
-# Copyright © 2017-2018 Unrud <unrud@outlook.com>
+# Copyright © 2017-2022 Unrud <unrud@outlook.com>
 #
 # This library is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

radicale/storage/multifilesystem/__init__.py

@@ -1,7 +1,8 @@
 # This file is part of Radicale - CalDAV and CardDAV server
 # Copyright © 2014 Jean-Marc Martins
 # Copyright © 2012-2017 Guillaume Ayoub
-# Copyright © 2017-2019 Unrud <unrud@outlook.com>
+# Copyright © 2017-2021 Unrud <unrud@outlook.com>
+# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
 #
 # This library is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -28,6 +29,7 @@ import time
 from typing import ClassVar, Iterator, Optional, Type
 
 from radicale import config
+from radicale.log import logger
 from radicale.storage.multifilesystem.base import CollectionBase, StorageBase
 from radicale.storage.multifilesystem.cache import CollectionPartCache
 from radicale.storage.multifilesystem.create_collection import \
@@ -89,3 +91,5 @@ class Storage(
     def __init__(self, configuration: config.Configuration) -> None:
         super().__init__(configuration)
         self._makedirs_synced(self._filesystem_folder)
+        logger.info("storage location: %r", self._filesystem_folder)
+        logger.info("storage cache subfolder usage for item: %s", self._use_cache_subfolder_for_item)

radicale/storage/multifilesystem/base.py

@@ -70,6 +70,7 @@ class StorageBase(storage.BaseStorage):
 
     _filesystem_folder: str
     _filesystem_fsync: bool
+    _use_cache_subfolder_for_item: bool
 
     def __init__(self, configuration: config.Configuration) -> None:
         super().__init__(configuration)
@@ -77,10 +78,17 @@ class StorageBase(storage.BaseStorage):
             "storage", "filesystem_folder")
         self._filesystem_fsync = configuration.get(
             "storage", "_filesystem_fsync")
+        self._use_cache_subfolder_for_item = configuration.get(
+            "storage", "use_cache_subfolder_for_item")
 
     def _get_collection_root_folder(self) -> str:
         return os.path.join(self._filesystem_folder, "collection-root")
 
+    def _get_collection_cache_folder(self, path, folder, subfolder) -> str:
+        if (self._use_cache_subfolder_for_item is True) and (subfolder == "item"):
+            path = path.replace(os.path.join(self._filesystem_folder, "collection-root"), os.path.join(self._filesystem_folder, "collection-cache"))
+        return os.path.join(path, folder, subfolder)
+
     def _fsync(self, f: IO[AnyStr]) -> None:
         if self._filesystem_fsync:
             try:

radicale/storage/multifilesystem/cache.py

@@ -1,7 +1,8 @@
 # This file is part of Radicale - CalDAV and CardDAV server
 # Copyright © 2014 Jean-Marc Martins
 # Copyright © 2012-2017 Guillaume Ayoub
-# Copyright © 2017-2018 Unrud <unrud@outlook.com>
+# Copyright © 2017-2021 Unrud <unrud@outlook.com>
+# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
 #
 # This library is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -81,8 +82,7 @@ class CollectionPartCache(CollectionBase):
         if not cache_hash:
             cache_hash = self._item_cache_hash(
                 item.serialize().encode(self._encoding))
-        cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
+        cache_folder = self._storage._get_collection_cache_folder(self._filesystem_path, ".Radicale.cache", "item")
-                                    "item")
         content = self._item_cache_content(item)
         self._storage._makedirs_synced(cache_folder)
         # Race: Other processes might have created and locked the file.
@@ -95,8 +95,7 @@ class CollectionPartCache(CollectionBase):
 
     def _load_item_cache(self, href: str, cache_hash: str
                          ) -> Optional[CacheContent]:
-        cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
+        cache_folder = self._storage._get_collection_cache_folder(self._filesystem_path, ".Radicale.cache", "item")
-                                    "item")
         try:
             with open(os.path.join(cache_folder, href), "rb") as f:
                 hash_, *remainder = pickle.load(f)
@@ -110,8 +109,7 @@ class CollectionPartCache(CollectionBase):
         return None
 
     def _clean_item_cache(self) -> None:
-        cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
+        cache_folder = self._storage._get_collection_cache_folder(self._filesystem_path, ".Radicale.cache", "item")
-                                    "item")
         self._clean_cache(cache_folder, (
             e.name for e in os.scandir(cache_folder) if not
             os.path.isfile(os.path.join(self._filesystem_path, e.name))))

radicale/storage/multifilesystem/move.py

@@ -1,7 +1,8 @@
 # This file is part of Radicale - CalDAV and CardDAV server
 # Copyright © 2014 Jean-Marc Martins
 # Copyright © 2012-2017 Guillaume Ayoub
-# Copyright © 2017-2018 Unrud <unrud@outlook.com>
+# Copyright © 2017-2021 Unrud <unrud@outlook.com>
+# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
 #
 # This library is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -41,10 +42,8 @@ class StoragePartMove(StorageBase):
         if item.collection._filesystem_path != to_collection._filesystem_path:
             self._sync_directory(item.collection._filesystem_path)
         # Move the item cache entry
-        cache_folder = os.path.join(item.collection._filesystem_path,
+        cache_folder = self._get_collection_cache_folder(item.collection._filesystem_path, ".Radicale.cache", "item")
-                                    ".Radicale.cache", "item")
+        to_cache_folder = self._get_collection_cache_folder(to_collection._filesystem_path, ".Radicale.cache", "item")
-        to_cache_folder = os.path.join(to_collection._filesystem_path,
-                                       ".Radicale.cache", "item")
         self._makedirs_synced(to_cache_folder)
         try:
             os.replace(os.path.join(cache_folder, item.href),

radicale/storage/multifilesystem/upload.py

@@ -1,7 +1,8 @@
 # This file is part of Radicale - CalDAV and CardDAV server
 # Copyright © 2014 Jean-Marc Martins
 # Copyright © 2012-2017 Guillaume Ayoub
-# Copyright © 2017-2018 Unrud <unrud@outlook.com>
+# Copyright © 2017-2022 Unrud <unrud@outlook.com>
+# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
 #
 # This library is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -75,8 +76,7 @@ class CollectionPartUpload(CollectionPartGet, CollectionPartCache,
             yield radicale_item.find_available_uid(
                 lambda href: not is_safe_free_href(href), suffix)
 
-        cache_folder = os.path.join(self._filesystem_path,
+        cache_folder = self._storage._get_collection_cache_folder(self._filesystem_path, ".Radicale.cache", "item")
-                                    ".Radicale.cache", "item")
         self._storage._makedirs_synced(cache_folder)
         for item in items:
             uid = item.uid

radicale/tests/test_storage.py

@@ -1,6 +1,7 @@
 # This file is part of Radicale - CalDAV and CardDAV server
 # Copyright © 2012-2017 Guillaume Ayoub
-# Copyright © 2017-2019 Unrud <unrud@outlook.com>
+# Copyright © 2017-2022 Unrud <unrud@outlook.com>
+# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
 #
 # This library is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -99,6 +100,22 @@ class TestMultiFileSystem(BaseTest):
         assert answer1 == answer2
         assert os.path.exists(os.path.join(cache_folder, "event1.ics"))
 
+    def test_item_cache_rebuild_subfolder(self) -> None:
+        """Delete the item cache and verify that it is rebuild."""
+        self.configure({"storage": {"use_cache_subfolder_for_item": "True"}})
+        self.mkcalendar("/calendar.ics/")
+        event = get_file_content("event1.ics")
+        path = "/calendar.ics/event1.ics"
+        self.put(path, event)
+        _, answer1 = self.get(path)
+        cache_folder = os.path.join(self.colpath, "collection-cache",
+                                    "calendar.ics", ".Radicale.cache", "item")
+        assert os.path.exists(os.path.join(cache_folder, "event1.ics"))
+        shutil.rmtree(cache_folder)
+        _, answer2 = self.get(path)
+        assert answer1 == answer2
+        assert os.path.exists(os.path.join(cache_folder, "event1.ics"))
+
     def test_put_whole_calendar_uids_used_as_file_names(self) -> None:
         """Test if UIDs are used as file names."""
         _TestBaseRequests.test_put_whole_calendar(