diff --git a/radicale/__init__.py b/radicale/__init__.py index 4a74581c..042c3242 100644 --- a/radicale/__init__.py +++ b/radicale/__init__.py @@ -24,6 +24,7 @@ Can be used with an external WSGI server or the built-in server. """ import base64 +import contextlib import datetime import io import itertools @@ -34,6 +35,7 @@ import posixpath import pprint import random import socket +import sys import threading import time import zlib @@ -545,6 +547,16 @@ class Application: except socket.timeout as e: logger.debug("client timed out", exc_info=True) return REQUEST_TIMEOUT + # Prepare before locking + props = xmlutils.props_from_request(xml_content) + props["tag"] = "VCALENDAR" + # TODO: use this? + # timezone = props.get("C:calendar-timezone") + try: + storage.check_and_sanitize_props(props) + except ValueError as e: + logger.warning( + "Bad MKCALENDAR request on %r: %s", path, e, exc_info=True) with self.Collection.acquire_lock("w", user): item = next(self.Collection.discover(path), None) if item: @@ -558,12 +570,7 @@ class Application: if (not isinstance(parent_item, storage.BaseCollection) or parent_item.get_meta("tag")): return FORBIDDEN - props = xmlutils.props_from_request(xml_content) - props["tag"] = "VCALENDAR" - # TODO: use this? - # timezone = props.get("C:calendar-timezone") try: - storage.check_and_sanitize_props(props) self.Collection.create_collection(path, props=props) except ValueError as e: logger.warning( @@ -585,6 +592,17 @@ class Application: except socket.timeout as e: logger.debug("client timed out", exc_info=True) return REQUEST_TIMEOUT + # Prepare before locking + props = xmlutils.props_from_request(xml_content) + try: + storage.check_and_sanitize_props(props) + except ValueError as e: + logger.warning( + "Bad MKCOL request on %r: %s", path, e, exc_info=True) + return BAD_REQUEST + if (props.get("tag") and "w" not in permissions or + not props.get("tag") and "W" not in permissions): + return NOT_ALLOWED with self.Collection.acquire_lock("w", user): item = next(self.Collection.discover(path), None) if item: @@ -597,12 +615,7 @@ class Application: if (not isinstance(parent_item, storage.BaseCollection) or parent_item.get_meta("tag")): return FORBIDDEN - props = xmlutils.props_from_request(xml_content) - if (props.get("tag") and "w" not in permissions or - not props.get("tag") and "W" not in permissions): - return NOT_ALLOWED try: - storage.check_and_sanitize_props(props) self.Collection.create_collection(path, props=props) except ValueError as e: logger.warning( @@ -755,9 +768,110 @@ class Application: except socket.timeout as e: logger.debug("client timed out", exc_info=True) return REQUEST_TIMEOUT + # Prepare before locking + parent_path = storage.sanitize_path( + "/%s/" % posixpath.dirname(path.strip("/"))) + permissions = self.Rights.authorized(user, path, "Ww") + parent_permissions = self.Rights.authorized(user, parent_path, "w") + + def prepare(vobject_items, tag=None, write_whole_collection=None): + if (write_whole_collection or + permissions and not parent_permissions): + write_whole_collection = True + content_type = environ.get("CONTENT_TYPE", + "").split(";")[0] + tags = {value: key + for key, value in xmlutils.MIMETYPES.items()} + tag = storage.predict_tag_of_whole_collection( + vobject_items, tags.get(content_type)) + if not tag: + raise ValueError("Can't determine collection tag") + collection_path = storage.sanitize_path(path).strip("/") + elif (write_whole_collection is not None and + not write_whole_collection or + not permissions and parent_permissions): + write_whole_collection = False + if tag is None: + tag = storage.predict_tag_of_parent_collection( + vobject_items) + collection_path = posixpath.dirname( + storage.sanitize_path(path).strip("/")) + props = None + stored_exc_info = None + items = [] + try: + if tag: + storage.check_and_sanitize_items( + vobject_items, is_collection=write_whole_collection, + tag=tag) + if write_whole_collection and tag == "VCALENDAR": + vobject_components = [] + vobject_item, = vobject_items + for content in ("vevent", "vtodo", "vjournal"): + vobject_components.extend( + getattr(vobject_item, "%s_list" % content, [])) + vobject_components_by_uid = itertools.groupby( + sorted(vobject_components, key=storage.get_uid), + storage.get_uid) + for uid, components in vobject_components_by_uid: + vobject_collection = vobject.iCalendar() + for component in components: + vobject_collection.add(component) + item = storage.Item( + collection_path=collection_path, + item=vobject_collection) + item.prepare() + items.append(item) + elif write_whole_collection and tag == "VADDRESSBOOK": + for vobject_item in vobject_items: + item = storage.Item( + collection_path=collection_path, + item=vobject_item) + item.prepare() + items.append(item) + elif not write_whole_collection: + vobject_item, = vobject_items + item = storage.Item(collection_path=collection_path, + item=vobject_item) + item.prepare() + items.append(item) + + if write_whole_collection: + props = {} + if tag: + props["tag"] = tag + if tag == "VCALENDAR" and vobject_items: + if hasattr(vobject_items[0], "x_wr_calname"): + calname = vobject_items[0].x_wr_calname.value + if calname: + props["D:displayname"] = calname + if hasattr(vobject_items[0], "x_wr_caldesc"): + caldesc = vobject_items[0].x_wr_caldesc.value + if caldesc: + props["C:calendar-description"] = caldesc + storage.check_and_sanitize_props(props) + except Exception: + stored_exc_info = sys.exc_info() + + # Use generator for items and delete references to free memory + # early + def items_generator(): + while items: + yield items.pop(0) + + return (items_generator(), tag, write_whole_collection, props, + stored_exc_info) + + try: + vobject_items = tuple(vobject.readComponents(content or "")) + except Exception as e: + logger.warning( + "Bad PUT request on %r: %s", path, e, exc_info=True) + return BAD_REQUEST + (prepared_items, prepared_tag, prepared_write_whole_collection, + prepared_props, prepared_exc_info) = prepare(vobject_items) + with self.Collection.acquire_lock("w", user): - parent_path = storage.sanitize_path( - "/%s/" % posixpath.dirname(path.strip("/"))) item = next(self.Collection.discover(path), None) parent_item = next(self.Collection.discover(parent_path), None) if not parent_item: @@ -766,8 +880,14 @@ class Application: write_whole_collection = ( isinstance(item, storage.BaseCollection) or not parent_item.get_meta("tag")) + if write_whole_collection: - if not self.Rights.authorized(user, path, "w"): + tag = prepared_tag + else: + tag = parent_item.get_meta("tag") + + if write_whole_collection: + if not self.Rights.authorized(user, path, "w" if tag else "W"): return NOT_ALLOWED elif not self.Rights.authorized(user, parent_path, "w"): return NOT_ALLOWED @@ -785,69 +905,43 @@ class Application: # Creation asked but item found: item can't be replaced return PRECONDITION_FAILED - try: - items = tuple(vobject.readComponents(content or "")) - if write_whole_collection: - content_type = environ.get("CONTENT_TYPE", - "").split(";")[0] - tags = {value: key - for key, value in xmlutils.MIMETYPES.items()} - tag = tags.get(content_type) - if items and items[0].name == "VCALENDAR": - tag = "VCALENDAR" - elif items and items[0].name in ("VCARD", "VLIST"): - tag = "VADDRESSBOOK" - elif not tag and not items: - # Maybe an empty address book - tag = "VADDRESSBOOK" - elif not tag: - raise ValueError("Can't determine collection tag") - else: - tag = parent_item.get_meta("tag") - storage.check_and_sanitize_items( - items, is_collection=write_whole_collection, tag=tag) - except Exception as e: + if (tag != prepared_tag or + prepared_write_whole_collection != write_whole_collection): + (prepared_items, prepared_tag, prepared_write_whole_collection, + prepared_props, prepared_exc_info) = prepare( + vobject_items, tag, write_whole_collection) + props = prepared_props + if prepared_exc_info: logger.warning( - "Bad PUT request on %r: %s", path, e, exc_info=True) + "Bad PUT request on %r: %s", path, prepared_exc_info[1], + exc_info=prepared_exc_info) return BAD_REQUEST if write_whole_collection: - props = {} - if tag: - props["tag"] = tag - if tag == "VCALENDAR" and items: - if hasattr(items[0], "x_wr_calname"): - calname = items[0].x_wr_calname.value - if calname: - props["D:displayname"] = calname - if hasattr(items[0], "x_wr_caldesc"): - caldesc = items[0].x_wr_caldesc.value - if caldesc: - props["C:calendar-description"] = caldesc try: - storage.check_and_sanitize_props(props) - new_item = self.Collection.create_collection( - path, items, props) + etag = self.Collection.create_collection( + path, prepared_items, props).etag except ValueError as e: logger.warning( "Bad PUT request on %r: %s", path, e, exc_info=True) return BAD_REQUEST else: - uid = storage.get_uid_from_object(items[0]) - if (item and item.uid != uid or - not item and parent_item.has_uid(uid)): + prepared_item, = prepared_items + if (item and item.uid != prepared_item.uid or + not item and parent_item.has_uid(prepared_item.uid)): return self._webdav_error_response( "C" if tag == "VCALENDAR" else "CR", "no-uid-conflict") href = posixpath.basename(path.strip("/")) try: - new_item = parent_item.upload(href, items[0]) + etag = parent_item.upload(href, prepared_item).etag except ValueError as e: logger.warning( "Bad PUT request on %r: %s", path, e, exc_info=True) return BAD_REQUEST - headers = {"ETag": new_item.etag} + + headers = {"ETag": etag} return client.CREATED, headers, None def do_REPORT(self, environ, base_prefix, path, user): @@ -863,7 +957,8 @@ class Application: except socket.timeout as e: logger.debug("client timed out", exc_info=True) return REQUEST_TIMEOUT - with self.Collection.acquire_lock("r", user): + with contextlib.ExitStack() as lock_stack: + lock_stack.enter_context(self.Collection.acquire_lock("r", user)) item = next(self.Collection.discover(path), None) if not item: return NOT_FOUND @@ -876,7 +971,8 @@ class Application: headers = {"Content-Type": "text/xml; charset=%s" % self.encoding} try: status, xml_answer = xmlutils.report( - base_prefix, path, xml_content, collection) + base_prefix, path, xml_content, collection, + lock_stack.close) except ValueError as e: logger.warning( "Bad REPORT request on %r: %s", path, e, exc_info=True) diff --git a/radicale/storage.py b/radicale/storage.py index fc6b1f9d..fa27261d 100644 --- a/radicale/storage.py +++ b/radicale/storage.py @@ -39,7 +39,7 @@ import time from contextlib import contextmanager from hashlib import md5 from importlib import import_module -from itertools import chain, groupby +from itertools import chain from random import getrandbits from tempfile import NamedTemporaryFile, TemporaryDirectory @@ -115,6 +115,27 @@ def load(configuration): return CollectionCopy +def predict_tag_of_parent_collection(vobject_items): + if len(vobject_items) != 1: + return "" + if vobject_items[0].name == "VCALENDAR": + return "VCALENDAR" + if vobject_items[0].name in ("VCARD", "VLIST"): + return "VADDRESSBOOK" + return "" + + +def predict_tag_of_whole_collection(vobject_items, fallback_tag=None): + if vobject_items and vobject_items[0].name == "VCALENDAR": + return "VCALENDAR" + if vobject_items and vobject_items[0].name in ("VCARD", "VLIST"): + return "VADDRESSBOOK" + if not fallback_tag and not vobject_items: + # Maybe an empty address book + return "VADDRESSBOOK" + return fallback_tag + + def check_and_sanitize_items(vobject_items, is_collection=False, tag=None): """Check vobject items for common errors and add missing UIDs. @@ -360,12 +381,15 @@ class ComponentNotFoundError(ValueError): class Item: - def __init__(self, collection, item=None, href=None, last_modified=None, - text=None, etag=None, uid=None, name=None, - component_name=None): + def __init__(self, collection_path=None, collection=None, item=None, + href=None, last_modified=None, text=None, etag=None, uid=None, + name=None, component_name=None, time_range=None): """Initialize an item. - ``collection`` the parent collection. + ``collection_path`` the path of the parent collection (optional if + ``collection`` is set). + + ``collection`` the parent collection (optional). ``href`` the href of the item. @@ -380,9 +404,23 @@ class Item: ``uid`` the UID of the object (optional). See ``get_uid_from_object``. + ``name`` the name of the item (optional). See ``vobject_item.name``. + + ``component_name`` the name of the primary component (optional). + See ``find_tag``. + + ``time_range`` the enclosing time range. + See ``find_tag_and_time_range``. + """ if text is None and item is None: raise ValueError("at least one of 'text' or 'item' must be set") + if collection_path is None: + if collection is None: + raise ValueError("at least one of 'collection_path' or " + "'collection' must be set") + collection_path = collection.path + self._collection_path = collection_path self.collection = collection self.href = href self.last_modified = last_modified @@ -392,6 +430,7 @@ class Item: self._uid = uid self._name = name self._component_name = component_name + self._time_range = time_range def __getattr__(self, attr): return getattr(self.item, attr) @@ -402,7 +441,8 @@ class Item: self._text = self.item.serialize() except Exception as e: raise RuntimeError("Failed to serialize item %r from %r: %s" % - (self.href, self.collection.path, e)) from e + (self.href, self._collection_path, + e)) from e return self._text @property @@ -412,7 +452,8 @@ class Item: self._item = vobject.readOne(self._text) except Exception as e: raise RuntimeError("Failed to parse item %r from %r: %s" % - (self.href, self.collection.path, e)) from e + (self.href, self._collection_path, + e)) from e return self._item @property @@ -430,9 +471,9 @@ class Item: @property def name(self): - if self._name is not None: - return self._name - return self.item.name + if self._name is None: + self._name = self.item.name or "" + return self._name @property def component_name(self): @@ -440,6 +481,24 @@ class Item: return self._component_name return xmlutils.find_tag(self.item) + @property + def time_range(self): + if self._time_range is None: + self._component_name, *self._time_range = ( + xmlutils.find_tag_and_time_range(self.item)) + return self._time_range + + def prepare(self): + """Fill cache with values.""" + orig_item = self._item + self.serialize() + self.etag + self.uid + self.name + self.time_range + self.component_name + self._item = orig_item + class BaseCollection: @@ -497,7 +556,7 @@ class BaseCollection: """ if item.collection.path == to_collection.path and item.href == to_href: return - to_collection.upload(to_href, item.item) + to_collection.upload(to_href, item) item.collection.delete(item.href) @property @@ -510,7 +569,7 @@ class BaseCollection: return '"%s"' % etag.hexdigest() @classmethod - def create_collection(cls, href, collection=None, props=None): + def create_collection(cls, href, items=None, props=None): """Create a collection. ``href`` is the sanitized path. @@ -608,7 +667,7 @@ class BaseCollection: return True return False - def upload(self, href, vobject_item): + def upload(self, href, item): """Upload a new or replace an existing item.""" raise NotImplementedError @@ -931,7 +990,7 @@ class Collection(BaseCollection): return item_errors == 0 and collection_errors == 0 @classmethod - def create_collection(cls, href, collection=None, props=None): + def create_collection(cls, href, items=None, props=None): folder = cls._get_collection_root_folder() # Path should already be sanitized @@ -953,25 +1012,11 @@ class Collection(BaseCollection): os.makedirs(tmp_filesystem_path) self = cls(sane_path, filesystem_path=tmp_filesystem_path) self.set_meta(props) - - if collection: + if items is not None: if props.get("tag") == "VCALENDAR": - collection, = collection - items = [] - for content in ("vevent", "vtodo", "vjournal"): - items.extend( - getattr(collection, "%s_list" % content, [])) - items_by_uid = groupby(sorted(items, key=get_uid), get_uid) - - def vobject_items(): - for uid, items in items_by_uid: - collection = vobject.iCalendar() - for item in items: - collection.add(item) - yield collection - self._upload_all_nonatomic(vobject_items(), suffix=".ics") + self._upload_all_nonatomic(items, suffix=".ics") elif props.get("tag") == "VADDRESSBOOK": - self._upload_all_nonatomic(collection, suffix=".vcf") + self._upload_all_nonatomic(items, suffix=".vcf") # This operation is not atomic on the filesystem level but it's # very unlikely that one rename operations succeeds while the @@ -983,7 +1028,7 @@ class Collection(BaseCollection): return cls(sane_path) - def _upload_all_nonatomic(self, vobject_items, suffix=""): + def _upload_all_nonatomic(self, items, suffix=""): """Upload a new set of items. This takes a list of vobject items and @@ -994,11 +1039,10 @@ class Collection(BaseCollection): ".Radicale.cache", "item") self._makedirs_synced(cache_folder) hrefs = set() - for vobject_item in vobject_items: - uid = get_uid_from_object(vobject_item) + for item in items: + uid = item.uid try: - cache_content = self._item_cache_content(vobject_item) - _, _, _, text, _, _, _, _ = cache_content + cache_content = self._item_cache_content(item) except Exception as e: raise ValueError( "Failed to store item %r in temporary collection %r: %s" % @@ -1036,7 +1080,7 @@ class Collection(BaseCollection): with self._atomic_write(os.path.join(self._filesystem_path, "ign"), newline="", sync_directory=False, replace_fn=replace_fn) as f: - f.write(text) + f.write(item.serialize()) hrefs.add(href) with self._atomic_write(os.path.join(cache_folder, href), "wb", sync_directory=False) as f: @@ -1267,30 +1311,23 @@ class Collection(BaseCollection): continue yield href - def get(self, href, verify_href=True): - item, metadata = self._get_with_metadata(href, verify_href=verify_href) - return item - def _item_cache_hash(self, raw_text): _hash = md5() _hash.update(ITEM_CACHE_TAG) _hash.update(raw_text) return _hash.hexdigest() - def _item_cache_content(self, vobject_item, cache_hash=None): - text = vobject_item.serialize() + def _item_cache_content(self, item, cache_hash=None): + text = item.serialize() if cache_hash is None: cache_hash = self._item_cache_hash(text.encode(self._encoding)) - etag = get_etag(text) - uid = get_uid_from_object(vobject_item) - name = vobject_item.name - tag, start, end = xmlutils.find_tag_and_time_range(vobject_item) - return cache_hash, uid, etag, text, name, tag, start, end + return (cache_hash, item.uid, item.etag, text, item.name, + item.component_name, *item.time_range) - def _store_item_cache(self, href, vobject_item, cache_hash=None): + def _store_item_cache(self, href, item, cache_hash=None): cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache", "item") - content = self._item_cache_content(vobject_item, cache_hash) + content = self._item_cache_content(item, cache_hash) self._makedirs_synced(cache_folder) try: # Race: Other processes might have created and locked the @@ -1335,10 +1372,7 @@ class Collection(BaseCollection): e.name for e in os.scandir(cache_folder) if not os.path.isfile(os.path.join(self._filesystem_path, e.name)))) - def _get_with_metadata(self, href, verify_href=True): - """Like ``get`` but additonally returns the following metadata: - tag, start, end: see ``xmlutils.find_tag_and_time_range``. If - extraction of the metadata failed, the values are all ``None``.""" + def get(self, href, verify_href=True): if verify_href: try: if not is_safe_filesystem_path_component(href): @@ -1348,26 +1382,25 @@ class Collection(BaseCollection): logger.debug( "Can't translate name %r safely to filesystem in %r: %s", href, self.path, e, exc_info=True) - return None, None + return None else: path = os.path.join(self._filesystem_path, href) try: with open(path, "rb") as f: raw_text = f.read() except (FileNotFoundError, IsADirectoryError): - return None, None + return None except PermissionError: # Windows raises ``PermissionError`` when ``path`` is a directory if (os.name == "nt" and os.path.isdir(path) and os.access(path, os.R_OK)): - return None, None + return None raise # The hash of the component in the file system. This is used to check, # if the entry in the cache is still valid. input_hash = self._item_cache_hash(raw_text) cache_hash, uid, etag, text, name, tag, start, end = \ self._load_item_cache(href, input_hash) - vobject_item = None if input_hash != cache_hash: with self._acquire_cache_lock("item"): # Lock the item cache to prevent multpile processes from @@ -1383,10 +1416,11 @@ class Collection(BaseCollection): raw_text.decode(self._encoding))) check_and_sanitize_items(vobject_items, tag=self.get_meta("tag")) - vobject_item = vobject_items[0] + vobject_item, = vobject_items + temp_item = Item(collection=self, item=vobject_item) cache_hash, uid, etag, text, name, tag, start, end = \ self._store_item_cache( - href, vobject_item, input_hash) + href, temp_item, input_hash) except Exception as e: raise RuntimeError("Failed to load item %r in %r: %s" % (href, self.path, e)) from e @@ -1398,10 +1432,12 @@ class Collection(BaseCollection): last_modified = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(os.path.getmtime(path))) + # Don't keep reference to ``vobject_item``, because it requires a lot + # of memory. return Item( - self, href=href, last_modified=last_modified, etag=etag, - text=text, item=vobject_item, uid=uid, name=name, - component_name=tag), (tag, start, end) + collection=self, href=href, last_modified=last_modified, etag=etag, + text=text, uid=uid, name=name, component_name=tag, + time_range=(start, end)) def get_multi(self, hrefs): # It's faster to check for file name collissions here, because @@ -1433,33 +1469,29 @@ class Collection(BaseCollection): # no filter yield from ((item, simple) for item in self.get_all()) return - for item, (itag, istart, iend) in ( - self._get_with_metadata(href, verify_href=False) - for href in self.list()): - if tag == itag and istart < end and iend > start: + for item in (self.get(h, verify_href=False) for h in self.list()): + istart, iend = item.time_range + if tag == item.component_name and istart < end and iend > start: yield item, simple and (start <= istart or iend <= end) - def upload(self, href, vobject_item): + def upload(self, href, item): if not is_safe_filesystem_path_component(href): raise UnsafePathError(href) try: - cache_hash, uid, etag, text, name, tag, _, _ = \ - self._store_item_cache(href, vobject_item) + self._store_item_cache(href, item) except Exception as e: raise ValueError("Failed to store item %r in collection %r: %s" % (href, self.path, e)) from e path = path_to_filesystem(self._filesystem_path, href) with self._atomic_write(path, newline="") as fd: - fd.write(text) + fd.write(item.serialize()) # Clean the cache after the actual item is stored, or the cache entry # will be removed again. self._clean_item_cache() - item = Item(self, href=href, etag=etag, text=text, item=vobject_item, - uid=uid, name=name, component_name=tag) # Track the change self._update_history_etag(href, item) self._clean_history_cache() - return item + return self.get(href, verify_href=False) def delete(self, href=None): if href is None: diff --git a/radicale/xmlutils.py b/radicale/xmlutils.py index 0b7b3f09..ceb23798 100644 --- a/radicale/xmlutils.py +++ b/radicale/xmlutils.py @@ -653,8 +653,8 @@ def find_tag(vobject_item): if vobject_item.name == "VCALENDAR": for component in vobject_item.components(): if component.name != "VTIMEZONE": - return component.name - return None + return component.name or "" + return "" def find_tag_and_time_range(vobject_item): @@ -668,7 +668,7 @@ def find_tag_and_time_range(vobject_item): """ tag = find_tag(vobject_item) if not tag: - return (None, TIMESTAMP_MIN, TIMESTAMP_MAX) + return (tag, TIMESTAMP_MIN, TIMESTAMP_MAX) start = end = None def range_fn(range_start, range_end, is_recurrence): @@ -1116,7 +1116,7 @@ def proppatch(base_prefix, path, xml_request, collection): return multistatus -def report(base_prefix, path, xml_request, collection): +def report(base_prefix, path, xml_request, collection, unlock_storage_fn): """Read and answer REPORT requests. Read rfc3253-3.6 for info. @@ -1230,8 +1230,15 @@ def report(base_prefix, path, xml_request, collection): if collection_requested: yield from collection.get_all_filtered(filters) + # Retrieve everything required for finishing the request. + retrieved_items = list(retrieve_items(collection, hreferences, + multistatus)) + collection_tag = collection.get_meta("tag") + # Don't access storage after this! + unlock_storage_fn() + def match(item, filter_): - tag = collection.get_meta("tag") + tag = collection_tag if (tag == "VCALENDAR" and filter_.tag != _tag("C", filter_)): if len(filter_) == 0: return True @@ -1253,8 +1260,11 @@ def report(base_prefix, path, xml_request, collection): return all(_prop_match(item.item, f, "CR") for f in filter_) raise ValueError("unsupported filter %r for %r" % (filter_.tag, tag)) - for item, filters_matched in retrieve_items(collection, hreferences, - multistatus): + while retrieved_items: + # ``item.item`` might be accessed during filtering. + # Don't keep reference to ``item``, because VObject requires a lot of + # memory. + item, filters_matched = retrieved_items.pop(0) if filters and not filters_matched: try: if not all(match(item, filter_) for filter_ in filters):