mirrors/Radicale
1
0
Fork 0
mirror of https://github.com/Kozea/Radicale.git synced 2025-04-04 21:57:43 +03:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #1642 from pbiering/storage-cache-separation

Browse source 
Storage cache separation
This commit is contained in:
Peter Bieringer 2024-12-03 21:21:13 +00:00 • committed by GitHub
commit 6943eb659f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 64 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -1,5 +1,9 @@
# Changelog
## 3.3.2.dev
* Fix: debug logging in rights/from_file
* Add: option [storage] use_cache_subfolder_for_item for storing item cache outside collection-root
## 3.3.1
* Add: option [auth] type=dovecot

6
DOCUMENTATION.md
View file

@ -1005,6 +1005,12 @@ Folder for storing local collections, created if not present.
Default: `/var/lib/radicale/collections`
##### use_cache_subfolder_for_item
Use subfolder `collections-cache' for cache file structure of item instead of inside collection folders, created if not present
Default: `False`
##### max_sync_token_age
Delete sync-token that are older than the specified time. (seconds)

3
config
View file

@ -138,6 +138,9 @@
# Folder for storing local collections, created if not present
#filesystem_folder = /var/lib/radicale/collections
# Use subfolder `collections-cache' for item cache file structure instead of inside collection folder
#use_cache_subfolder_for_item = False
# Delete sync token that are older (seconds)
#max_sync_token_age = 2592000

4
radicale/config.py
View file

@ -279,6 +279,10 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([
"value": "/var/lib/radicale/collections",
"help": "path where collections are stored",
"type": filepath}),
("use_cache_subfolder_for_item", {
"value": "False",
"help": "use subfolder `collections-cache' for item cache file structure instead of inside collection folder",
"type": bool}),
("max_sync_token_age", {
"value": "2592000", # 30 days
"help": "delete sync token that are older",

8
radicale/rights/from_file.py
View file

@ -1,6 +1,7 @@
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2019 Unrud <unrud@outlook.com>
# Copyright © 2017-2021 Unrud <unrud@outlook.com>
# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -99,12 +100,9 @@ class Rights(rights.BaseRights):
user, sane_path, user_pattern,
collection_pattern, section, permission)
return permission
logger.debug("Rule %r:%r doesn't match %r:%r from section %r",
user, sane_path, user_pattern, collection_pattern,
section)
if self._log_rights_rule_doesnt_match_on_debug:
logger.debug("Rule %r:%r doesn't match %r:%r from section %r",
user, sane_path, user_pattern, collection_pattern,
section)
logger.info("Rights: %r:%r doesn't match any section", user, sane_path)
logger.debug("Rights: %r:%r doesn't match any section", user, sane_path)
return ""

2
radicale/storage/__init__.py
View file

@ -1,7 +1,7 @@
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
# Copyright © 2017-2022 Unrud <unrud@outlook.com>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by

6
radicale/storage/multifilesystem/__init__.py
View file

@ -1,7 +1,8 @@
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2019 Unrud <unrud@outlook.com>
# Copyright © 2017-2021 Unrud <unrud@outlook.com>
# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -28,6 +29,7 @@ import time
from typing import ClassVar, Iterator, Optional, Type
from radicale import config
from radicale.log import logger
from radicale.storage.multifilesystem.base import CollectionBase, StorageBase
from radicale.storage.multifilesystem.cache import CollectionPartCache
from radicale.storage.multifilesystem.create_collection import \
@ -89,3 +91,5 @@ class Storage(
def __init__(self, configuration: config.Configuration) -> None:
super().__init__(configuration)
self._makedirs_synced(self._filesystem_folder)
logger.info("storage location: %r", self._filesystem_folder)
logger.info("storage cache subfolder usage for item: %s", self._use_cache_subfolder_for_item)

8
radicale/storage/multifilesystem/base.py
View file

@ -70,6 +70,7 @@ class StorageBase(storage.BaseStorage):
_filesystem_folder: str
_filesystem_fsync: bool
_use_cache_subfolder_for_item: bool
def __init__(self, configuration: config.Configuration) -> None:
super().__init__(configuration)
@ -77,10 +78,17 @@ class StorageBase(storage.BaseStorage):
"storage", "filesystem_folder")
self._filesystem_fsync = configuration.get(
"storage", "_filesystem_fsync")
self._use_cache_subfolder_for_item = configuration.get(
"storage", "use_cache_subfolder_for_item")
def _get_collection_root_folder(self) -> str:
return os.path.join(self._filesystem_folder, "collection-root")
def _get_collection_cache_folder(self, path, folder, subfolder) -> str:
if (self._use_cache_subfolder_for_item is True) and (subfolder == "item"):
path = path.replace(os.path.join(self._filesystem_folder, "collection-root"), os.path.join(self._filesystem_folder, "collection-cache"))
return os.path.join(path, folder, subfolder)
def _fsync(self, f: IO[AnyStr]) -> None:
if self._filesystem_fsync:
try:

12
radicale/storage/multifilesystem/cache.py
View file

@ -1,7 +1,8 @@
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
# Copyright © 2017-2021 Unrud <unrud@outlook.com>
# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -81,8 +82,7 @@ class CollectionPartCache(CollectionBase):
if not cache_hash:
cache_hash = self._item_cache_hash(
item.serialize().encode(self._encoding))
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
"item")
cache_folder = self._storage._get_collection_cache_folder(self._filesystem_path, ".Radicale.cache", "item")
content = self._item_cache_content(item)
self._storage._makedirs_synced(cache_folder)
# Race: Other processes might have created and locked the file.
@ -95,8 +95,7 @@ class CollectionPartCache(CollectionBase):
def _load_item_cache(self, href: str, cache_hash: str
) -> Optional[CacheContent]:
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
"item")
cache_folder = self._storage._get_collection_cache_folder(self._filesystem_path, ".Radicale.cache", "item")
try:
with open(os.path.join(cache_folder, href), "rb") as f:
hash_, *remainder = pickle.load(f)
@ -110,8 +109,7 @@ class CollectionPartCache(CollectionBase):
return None
def _clean_item_cache(self) -> None:
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
"item")
cache_folder = self._storage._get_collection_cache_folder(self._filesystem_path, ".Radicale.cache", "item")
self._clean_cache(cache_folder, (
e.name for e in os.scandir(cache_folder) if not
os.path.isfile(os.path.join(self._filesystem_path, e.name))))

9
radicale/storage/multifilesystem/move.py
View file

@ -1,7 +1,8 @@
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
# Copyright © 2017-2021 Unrud <unrud@outlook.com>
# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -41,10 +42,8 @@ class StoragePartMove(StorageBase):
if item.collection._filesystem_path != to_collection._filesystem_path:
self._sync_directory(item.collection._filesystem_path)
# Move the item cache entry
cache_folder = os.path.join(item.collection._filesystem_path,
".Radicale.cache", "item")
to_cache_folder = os.path.join(to_collection._filesystem_path,
".Radicale.cache", "item")
cache_folder = self._get_collection_cache_folder(item.collection._filesystem_path, ".Radicale.cache", "item")
to_cache_folder = self._get_collection_cache_folder(to_collection._filesystem_path, ".Radicale.cache", "item")
self._makedirs_synced(to_cache_folder)
try:
os.replace(os.path.join(cache_folder, item.href),

6
radicale/storage/multifilesystem/upload.py
View file

@ -1,7 +1,8 @@
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2014 Jean-Marc Martins
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2018 Unrud <unrud@outlook.com>
# Copyright © 2017-2022 Unrud <unrud@outlook.com>
# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -75,8 +76,7 @@ class CollectionPartUpload(CollectionPartGet, CollectionPartCache,
yield radicale_item.find_available_uid(
lambda href: not is_safe_free_href(href), suffix)
cache_folder = os.path.join(self._filesystem_path,
".Radicale.cache", "item")
cache_folder = self._storage._get_collection_cache_folder(self._filesystem_path, ".Radicale.cache", "item")
self._storage._makedirs_synced(cache_folder)
for item in items:
uid = item.uid

19
radicale/tests/test_storage.py
View file

@ -1,6 +1,7 @@
# This file is part of Radicale - CalDAV and CardDAV server
# Copyright © 2012-2017 Guillaume Ayoub
# Copyright © 2017-2019 Unrud <unrud@outlook.com>
# Copyright © 2017-2022 Unrud <unrud@outlook.com>
# Copyright © 2024-2024 Peter Bieringer <pb@bieringer.de>
#
# This library is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -99,6 +100,22 @@ class TestMultiFileSystem(BaseTest):
assert answer1 == answer2
assert os.path.exists(os.path.join(cache_folder, "event1.ics"))
def test_item_cache_rebuild_subfolder(self) -> None:
"""Delete the item cache and verify that it is rebuild."""
self.configure({"storage": {"use_cache_subfolder_for_item": "True"}})
self.mkcalendar("/calendar.ics/")
event = get_file_content("event1.ics")
path = "/calendar.ics/event1.ics"
self.put(path, event)
_, answer1 = self.get(path)
cache_folder = os.path.join(self.colpath, "collection-cache",
"calendar.ics", ".Radicale.cache", "item")
assert os.path.exists(os.path.join(cache_folder, "event1.ics"))
shutil.rmtree(cache_folder)
_, answer2 = self.get(path)
assert answer1 == answer2
assert os.path.exists(os.path.join(cache_folder, "event1.ics"))
def test_put_whole_calendar_uids_used_as_file_names(self) -> None:
"""Test if UIDs are used as file names."""
_TestBaseRequests.test_put_whole_calendar(