Improve free-busy report

radicale/app/report.py

@@ -33,15 +33,16 @@ import vobject.base
 from vobject.base import ContentLine
 
 import radicale.item as radicale_item
-from radicale import httputils, pathutils, storage, types, xmlutils
+from radicale import httputils, pathutils, storage, types, xmlutils, config
 from radicale.app.base import Access, ApplicationBase
 from radicale.item import filter as radicale_filter
 from radicale.log import logger
 
 def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Element],
-               collection: storage.BaseCollection, encoding: str,
+                     collection: storage.BaseCollection, encoding: str,
-               unlock_storage_fn: Callable[[], None]
+                     unlock_storage_fn: Callable[[], None],
-               ) -> Tuple[int, str]:
+                     max_occurrence: int
+                     ) -> Tuple[int, str]:
     multistatus = ET.Element(xmlutils.make_clark("D:multistatus"))
     if xml_request is None:
         return client.MULTI_STATUS, multistatus
@@ -53,16 +54,73 @@ def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Eleme
         return client.FORBIDDEN, xmlutils.webdav_error("D:supported-report")
 
     time_range_element = root.find(xmlutils.make_clark("C:time-range"))
-    start,end = radicale_filter.time_range_timestamps(time_range_element)
+
-    items = list(collection.get_by_time(start, end))
+    # Build a single filter from the free busy query for retrieval
+    # TODO: filter for VFREEBUSY in additional to VEVENT but
+    # test_filter doesn't support that yet.
+    vevent_cf_element = ET.Element(xmlutils.make_clark("C:comp-filter"),
+                                   attrib={'name':'VEVENT'})
+    vevent_cf_element.append(time_range_element)
+    vcalendar_cf_element = ET.Element(xmlutils.make_clark("C:comp-filter"),
+                                   attrib={'name':'VCALENDAR'})
+    vcalendar_cf_element.append(vevent_cf_element)
+    filter_element = ET.Element(xmlutils.make_clark("C:filter"))
+    filter_element.append(vcalendar_cf_element)
+    filters = (filter_element,)
+
+    # First pull from storage
+    retrieved_items = list(collection.get_filtered(filters))
+    # !!! Don't access storage after this !!!
+    unlock_storage_fn()
 
     cal = vobject.iCalendar()
-    for item in items:
+    collection_tag = collection.tag
-        occurrences = radicale_filter.time_range_fill(item.vobject_item, time_range_element, "VEVENT")
+    while retrieved_items:
+        # Second filtering before evaluating occurrences.
+        # ``item.vobject_item`` might be accessed during filtering.
+        # Don't keep reference to ``item``, because VObject requires a lot of
+        # memory.
+        item, filter_matched = retrieved_items.pop(0)
+        if not filter_matched:
+            try:
+                if not test_filter(collection_tag, item, filter_element):
+                    continue
+            except ValueError as e:
+                raise ValueError("Failed to free-busy filter item %r from %r: %s" %
+                                 (item.href, collection.path, e)) from e
+            except Exception as e:
+                raise RuntimeError("Failed to free-busy filter item %r from %r: %s" %
+                                   (item.href, collection.path, e)) from e
+
+        fbtype = None
+        if item.component_name == 'VEVENT':
+            transp = getattr(item.vobject_item, 'transp', None)
+            if transp and transp.value != 'OPAQUE':
+                continue
+
+            status = getattr(item.vobject_item, 'status', None)
+            if not status or status.value == 'CONFIRMED':
+                fbtype = 'BUSY'
+            elif status.value == 'CANCELLED':
+                fbtype = 'FREE'
+            elif status.value == 'TENTATIVE':
+                fbtype = 'BUSY-TENTATIVE'
+            else:
+                # Could do fbtype = status.value for x-name, I prefer this
+                fbtype = 'BUSY'
+
+        # TODO: coalesce overlapping periods
+
+        occurrences = radicale_filter.time_range_fill(item.vobject_item,
+                                                      time_range_element,
+                                                      "VEVENT",
+                                                      n=max_occurrence)
         for occurrence in occurrences:
             vfb = cal.add('vfreebusy')
             vfb.add('dtstamp').value = item.vobject_item.vevent.dtstamp.value
             vfb.add('dtstart').value, vfb.add('dtend').value = occurrence
+            if fbtype:
+                vfb.add('fbtype').value = fbtype
     return (client.OK, cal.serialize())
 
 
@@ -457,10 +515,11 @@ class ApplicationPartReport(ApplicationBase):
 
             if xml_content is not None and \
                xml_content.tag == xmlutils.make_clark("C:free-busy-query"):
+                max_occurrence = self.configuration.get("reporting", "max_freebusy_occurrence")
                 try:
                     status, body = free_busy_report(
                         base_prefix, path, xml_content, collection, self._encoding,
-                        lock_stack.close)
+                        lock_stack.close, max_occurrence)
                 except ValueError as e:
                     logger.warning(
                         "Bad REPORT request on %r: %s", path, e, exc_info=True)

radicale/config.py

@@ -293,8 +293,13 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([
             "help": "mask passwords in logs",
             "type": bool})])),
     ("headers", OrderedDict([
-        ("_allow_extra", str)]))])
+        ("_allow_extra", str)])),
-
+    ("reporting", OrderedDict([
+        ("max_freebusy_occurrence", {
+            "value": "10000",
+            "help": "number of occurrences per event when reporting",
+            "type": positive_int})]))
+    ])
 
 def parse_compound_paths(*compound_paths: Optional[str]
                          ) -> List[Tuple[str, bool]]:

radicale/storage/__init__.py

@@ -158,24 +158,6 @@ class BaseCollection:
                 continue
             yield item, simple and (start <= istart or iend <= end)
 
-    def get_by_time(self, start: int , end: int
-                    ) -> Iterable["radicale_item.Item"]:
-        """Fetch all items within a start and end time range.
-
-        Returns a iterable of ``item``s.
-
-        """
-        if not self.tag:
-            return
-        for item in self.get_all():
-            # TODO: Any other component_name here?
-            if item.component_name not in ("VEVENT",):
-                continue
-            istart, iend = item.time_range
-            if istart >= end or iend <= start:
-                continue
-            yield item
-
     def has_uid(self, uid: str) -> bool:
         """Check if a UID exists in the collection."""
         for item in self.get_all():

radicale/tests/test_base.py

@@ -1378,9 +1378,13 @@ permissions: RrWw""")
 <C:free-busy-query xmlns:C="urn:ietf:params:xml:ns:caldav">
     <C:time-range start="20130901T140000Z" end="20130908T220000Z"/>
 </C:free-busy-query>""", 200, is_xml = False)
-        assert len(responses) == 1
         for response in responses.values():
             assert isinstance(response, vobject.base.Component)
+        assert len(responses) == 1
+        vcalendar = list(responses.values())[0]
+        assert len(vcalendar.vfreebusy_list) == 2
+        for vfb in vcalendar.vfreebusy_list:
+            assert vfb.fbtype.value == 'BUSY'
 
     def _report_sync_token(
             self, calendar_path: str, sync_token: Optional[str] = None