Merge pull request #1337 from react0r-com/react0r

Add basic free/busy reporting

CHANGELOG.md

@@ -1,8 +1,12 @@
 # Changelog
 
 ## 3.dev
+* Fix: Using icalendar's tzinfo on created datetime to fix issue with icalendar
+* Enhancement: Added free-busy report
+* Enhancement: Added 'max_freebusy_occurrences` setting to avoid potential DOS on reports
 * Enhancement: remove unexpected control codes from uploaded items
 * Drop: remove unused requirement "typeguard"
+* Improve: Refactored some date parsing code
 
 ## 3.2.2
 * Enhancement: add support for auth.type=denyall (will be default for security reasons in upcoming releases)

DOCUMENTATION.md

@@ -1023,6 +1023,18 @@ RabbitMQ queue type for the topic.
 
 Default: classic
 
+#### reporting
+##### max_freebusy_occurrence
+
+When returning a free-busy report, a list of busy time occurrences are
+generated based on a given time frame. Large time frames could
+generate a lot of occurrences based on the time frame supplied. This
+setting limits the lookup to prevent potential denial of service
+attacks on large time frames. If the limit is reached, an HTTP error
+is thrown instead of returning the results.
+
+Default: 10000
+
 ## Supported Clients
 
 Radicale has been tested with:

config

@@ -172,3 +172,9 @@
 #rabbitmq_endpoint =
 #rabbitmq_topic =
 #rabbitmq_queue_type = classic
+
+[reporting]
+
+# When returning a free-busy report, limit the number of returned
+# occurences per event to prevent DOS attacks.
+#max_freebusy_occurrence = 10000

radicale/app/report.py

@@ -28,6 +28,7 @@ from typing import (Any, Callable, Iterable, Iterator, List, Optional,
                     Sequence, Tuple, Union)
 from urllib.parse import unquote, urlparse
 
+import vobject
 import vobject.base
 from vobject.base import ContentLine
 
@@ -38,11 +39,110 @@ from radicale.item import filter as radicale_filter
 from radicale.log import logger
 
 
+def free_busy_report(base_prefix: str, path: str, xml_request: Optional[ET.Element],
+                     collection: storage.BaseCollection, encoding: str,
+                     unlock_storage_fn: Callable[[], None],
+                     max_occurrence: int
+                     ) -> Tuple[int, Union[ET.Element, str]]:
+    # NOTE: this function returns both an Element and a string because
+    # free-busy reports are an edge-case on the return type according
+    # to the spec.
+
+    multistatus = ET.Element(xmlutils.make_clark("D:multistatus"))
+    if xml_request is None:
+        return client.MULTI_STATUS, multistatus
+    root = xml_request
+    if (root.tag == xmlutils.make_clark("C:free-busy-query") and
+            collection.tag != "VCALENDAR"):
+        logger.warning("Invalid REPORT method %r on %r requested",
+                       xmlutils.make_human_tag(root.tag), path)
+        return client.FORBIDDEN, xmlutils.webdav_error("D:supported-report")
+
+    time_range_element = root.find(xmlutils.make_clark("C:time-range"))
+    assert isinstance(time_range_element, ET.Element)
+
+    # Build a single filter from the free busy query for retrieval
+    # TODO: filter for VFREEBUSY in additional to VEVENT but
+    # test_filter doesn't support that yet.
+    vevent_cf_element = ET.Element(xmlutils.make_clark("C:comp-filter"),
+                                   attrib={'name': 'VEVENT'})
+    vevent_cf_element.append(time_range_element)
+    vcalendar_cf_element = ET.Element(xmlutils.make_clark("C:comp-filter"),
+                                      attrib={'name': 'VCALENDAR'})
+    vcalendar_cf_element.append(vevent_cf_element)
+    filter_element = ET.Element(xmlutils.make_clark("C:filter"))
+    filter_element.append(vcalendar_cf_element)
+    filters = (filter_element,)
+
+    # First pull from storage
+    retrieved_items = list(collection.get_filtered(filters))
+    # !!! Don't access storage after this !!!
+    unlock_storage_fn()
+
+    cal = vobject.iCalendar()
+    collection_tag = collection.tag
+    while retrieved_items:
+        # Second filtering before evaluating occurrences.
+        # ``item.vobject_item`` might be accessed during filtering.
+        # Don't keep reference to ``item``, because VObject requires a lot of
+        # memory.
+        item, filter_matched = retrieved_items.pop(0)
+        if not filter_matched:
+            try:
+                if not test_filter(collection_tag, item, filter_element):
+                    continue
+            except ValueError as e:
+                raise ValueError("Failed to free-busy filter item %r from %r: %s" %
+                                 (item.href, collection.path, e)) from e
+            except Exception as e:
+                raise RuntimeError("Failed to free-busy filter item %r from %r: %s" %
+                                   (item.href, collection.path, e)) from e
+
+        fbtype = None
+        if item.component_name == 'VEVENT':
+            transp = getattr(item.vobject_item.vevent, 'transp', None)
+            if transp and transp.value != 'OPAQUE':
+                continue
+
+            status = getattr(item.vobject_item.vevent, 'status', None)
+            if not status or status.value == 'CONFIRMED':
+                fbtype = 'BUSY'
+            elif status.value == 'CANCELLED':
+                fbtype = 'FREE'
+            elif status.value == 'TENTATIVE':
+                fbtype = 'BUSY-TENTATIVE'
+            else:
+                # Could do fbtype = status.value for x-name, I prefer this
+                fbtype = 'BUSY'
+
+        # TODO: coalesce overlapping periods
+
+        if max_occurrence > 0:
+            n_occurrences = max_occurrence+1
+        else:
+            n_occurrences = 0
+        occurrences = radicale_filter.time_range_fill(item.vobject_item,
+                                                      time_range_element,
+                                                      "VEVENT",
+                                                      n=n_occurrences)
+        if len(occurrences) >= max_occurrence:
+            raise ValueError("FREEBUSY occurrences limit of {} hit"
+                             .format(max_occurrence))
+
+        for occurrence in occurrences:
+            vfb = cal.add('vfreebusy')
+            vfb.add('dtstamp').value = item.vobject_item.vevent.dtstamp.value
+            vfb.add('dtstart').value, vfb.add('dtend').value = occurrence
+            if fbtype:
+                vfb.add('fbtype').value = fbtype
+    return (client.OK, cal.serialize())
+
+
 def xml_report(base_prefix: str, path: str, xml_request: Optional[ET.Element],
                collection: storage.BaseCollection, encoding: str,
                unlock_storage_fn: Callable[[], None]
                ) -> Tuple[int, ET.Element]:
-    """Read and answer REPORT requests.
+    """Read and answer REPORT requests that return XML.
 
     Read rfc3253-3.6 for info.
 
@@ -426,13 +526,28 @@ class ApplicationPartReport(ApplicationBase):
             else:
                 assert item.collection is not None
                 collection = item.collection
-            try:
+
-                status, xml_answer = xml_report(
+            if xml_content is not None and \
-                    base_prefix, path, xml_content, collection, self._encoding,
+               xml_content.tag == xmlutils.make_clark("C:free-busy-query"):
-                    lock_stack.close)
+                max_occurrence = self.configuration.get("reporting", "max_freebusy_occurrence")
-            except ValueError as e:
+                try:
-                logger.warning(
+                    status, body = free_busy_report(
-                    "Bad REPORT request on %r: %s", path, e, exc_info=True)
+                        base_prefix, path, xml_content, collection, self._encoding,
-                return httputils.BAD_REQUEST
+                        lock_stack.close, max_occurrence)
-        headers = {"Content-Type": "text/xml; charset=%s" % self._encoding}
+                except ValueError as e:
-        return status, headers, self._xml_response(xml_answer)
+                    logger.warning(
+                        "Bad REPORT request on %r: %s", path, e, exc_info=True)
+                    return httputils.BAD_REQUEST
+                headers = {"Content-Type": "text/calendar; charset=%s" % self._encoding}
+                return status, headers, str(body)
+            else:
+                try:
+                    status, xml_answer = xml_report(
+                        base_prefix, path, xml_content, collection, self._encoding,
+                        lock_stack.close)
+                except ValueError as e:
+                    logger.warning(
+                        "Bad REPORT request on %r: %s", path, e, exc_info=True)
+                    return httputils.BAD_REQUEST
+                headers = {"Content-Type": "text/xml; charset=%s" % self._encoding}
+                return status, headers, self._xml_response(xml_answer)

radicale/config.py

@@ -297,7 +297,13 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([
             "help": "mask passwords in logs",
             "type": bool})])),
     ("headers", OrderedDict([
-        ("_allow_extra", str)]))])
+        ("_allow_extra", str)])),
+    ("reporting", OrderedDict([
+        ("max_freebusy_occurrence", {
+            "value": "10000",
+            "help": "number of occurrences per event when reporting",
+            "type": positive_int})]))
+    ])
 
 
 def parse_compound_paths(*compound_paths: Optional[str]

radicale/item/filter.py

@@ -48,10 +48,34 @@ def date_to_datetime(d: date) -> datetime:
     if not isinstance(d, datetime):
         d = datetime.combine(d, datetime.min.time())
     if not d.tzinfo:
-        d = d.replace(tzinfo=timezone.utc)
+        # NOTE: using vobject's UTC as it wasn't playing well with datetime's.
+        d = d.replace(tzinfo=vobject.icalendar.utc)
     return d
 
 
+def parse_time_range(time_filter: ET.Element) -> Tuple[datetime, datetime]:
+    start_text = time_filter.get("start")
+    end_text = time_filter.get("end")
+    if start_text:
+        start = datetime.strptime(
+            start_text, "%Y%m%dT%H%M%SZ").replace(
+                tzinfo=timezone.utc)
+    else:
+        start = DATETIME_MIN
+    if end_text:
+        end = datetime.strptime(
+            end_text, "%Y%m%dT%H%M%SZ").replace(
+                tzinfo=timezone.utc)
+    else:
+        end = DATETIME_MAX
+    return start, end
+
+
+def time_range_timestamps(time_filter: ET.Element) -> Tuple[int, int]:
+    start, end = parse_time_range(time_filter)
+    return (math.floor(start.timestamp()), math.ceil(end.timestamp()))
+
+
 def comp_match(item: "item.Item", filter_: ET.Element, level: int = 0) -> bool:
     """Check whether the ``item`` matches the comp ``filter_``.
 
@@ -147,21 +171,10 @@ def time_range_match(vobject_item: vobject.base.Component,
     """Check whether the component/property ``child_name`` of
        ``vobject_item`` matches the time-range ``filter_``."""
 
-    start_text = filter_.get("start")
+    if not filter_.get("start") and not filter_.get("end"):
-    end_text = filter_.get("end")
-    if not start_text and not end_text:
         return False
-    if start_text:
-        start = datetime.strptime(start_text, "%Y%m%dT%H%M%SZ")
-    else:
-        start = datetime.min
-    if end_text:
-        end = datetime.strptime(end_text, "%Y%m%dT%H%M%SZ")
-    else:
-        end = datetime.max
-    start = start.replace(tzinfo=timezone.utc)
-    end = end.replace(tzinfo=timezone.utc)
 
+    start, end = parse_time_range(filter_)
     matched = False
 
     def range_fn(range_start: datetime, range_end: datetime,
@@ -181,6 +194,35 @@ def time_range_match(vobject_item: vobject.base.Component,
     return matched
 
 
+def time_range_fill(vobject_item: vobject.base.Component,
+                    filter_: ET.Element, child_name: str, n: int = 1
+                    ) -> List[Tuple[datetime, datetime]]:
+    """Create a list of ``n`` occurances from the component/property ``child_name``
+       of ``vobject_item``."""
+    if not filter_.get("start") and not filter_.get("end"):
+        return []
+
+    start, end = parse_time_range(filter_)
+    ranges: List[Tuple[datetime, datetime]] = []
+
+    def range_fn(range_start: datetime, range_end: datetime,
+                 is_recurrence: bool) -> bool:
+        nonlocal ranges
+        if start < range_end and range_start < end:
+            ranges.append((range_start, range_end))
+            if n > 0 and len(ranges) >= n:
+                return True
+        if end < range_start and not is_recurrence:
+            return True
+        return False
+
+    def infinity_fn(range_start: datetime) -> bool:
+        return False
+
+    visit_time_ranges(vobject_item, child_name, range_fn, infinity_fn)
+    return ranges
+
+
 def visit_time_ranges(vobject_item: vobject.base.Component, child_name: str,
                       range_fn: Callable[[datetime, datetime, bool], bool],
                       infinity_fn: Callable[[datetime], bool]) -> None:
@@ -543,20 +585,7 @@ def simplify_prefilters(filters: Iterable[ET.Element], collection_tag: str
                 if time_filter.tag != xmlutils.make_clark("C:time-range"):
                     simple = False
                     continue
-                start_text = time_filter.get("start")
+                start, end = time_range_timestamps(time_filter)
-                end_text = time_filter.get("end")
-                if start_text:
-                    start = math.floor(datetime.strptime(
-                        start_text, "%Y%m%dT%H%M%SZ").replace(
-                            tzinfo=timezone.utc).timestamp())
-                else:
-                    start = TIMESTAMP_MIN
-                if end_text:
-                    end = math.ceil(datetime.strptime(
-                        end_text, "%Y%m%dT%H%M%SZ").replace(
-                            tzinfo=timezone.utc).timestamp())
-                else:
-                    end = TIMESTAMP_MAX
                 return tag, start, end, simple
             return tag, TIMESTAMP_MIN, TIMESTAMP_MAX, simple
     return None, TIMESTAMP_MIN, TIMESTAMP_MAX, simple

radicale/tests/__init__.py

@@ -31,11 +31,12 @@ from io import BytesIO
 from typing import Any, Dict, List, Optional, Tuple, Union
 
 import defusedxml.ElementTree as DefusedET
+import vobject
 
 import radicale
 from radicale import app, config, types, xmlutils
 
-RESPONSES = Dict[str, Union[int, Dict[str, Tuple[int, ET.Element]]]]
+RESPONSES = Dict[str, Union[int, Dict[str, Tuple[int, ET.Element]], vobject.base.Component]]
 
 # Enable debug output
 radicale.log.logger.setLevel(logging.DEBUG)
@@ -107,8 +108,7 @@ class BaseTest:
     def parse_responses(text: str) -> RESPONSES:
         xml = DefusedET.fromstring(text)
         assert xml.tag == xmlutils.make_clark("D:multistatus")
-        path_responses: Dict[str, Union[
+        path_responses: RESPONSES = {}
-            int, Dict[str, Tuple[int, ET.Element]]]] = {}
         for response in xml.findall(xmlutils.make_clark("D:response")):
             href = response.find(xmlutils.make_clark("D:href"))
             assert href.text not in path_responses
@@ -133,6 +133,12 @@ class BaseTest:
                 path_responses[href.text] = prop_responses
         return path_responses
 
+    @staticmethod
+    def parse_free_busy(text: str) -> RESPONSES:
+        path_responses: RESPONSES = {}
+        path_responses[""] = vobject.readOne(text)
+        return path_responses
+
     def get(self, path: str, check: Optional[int] = 200, **kwargs
             ) -> Tuple[int, str]:
         assert "data" not in kwargs
@@ -177,13 +183,18 @@ class BaseTest:
         return status, responses
 
     def report(self, path: str, data: str, check: Optional[int] = 207,
+               is_xml: Optional[bool] = True,
                **kwargs) -> Tuple[int, RESPONSES]:
         status, _, answer = self.request("REPORT", path, data, check=check,
                                          **kwargs)
         if status < 200 or 300 <= status:
             return status, {}
         assert answer is not None
-        return status, self.parse_responses(answer)
+        if is_xml:
+            parsed = self.parse_responses(answer)
+        else:
+            parsed = self.parse_free_busy(answer)
+        return status, parsed
 
     def delete(self, path: str, check: Optional[int] = 200, **kwargs
                ) -> Tuple[int, RESPONSES]:

radicale/tests/static/event10.ics

@@ -0,0 +1,36 @@
+BEGIN:VCALENDAR
+PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN
+VERSION:2.0
+BEGIN:VTIMEZONE
+TZID:Europe/Paris
+X-LIC-LOCATION:Europe/Paris
+BEGIN:DAYLIGHT
+TZOFFSETFROM:+0100
+TZOFFSETTO:+0200
+TZNAME:CEST
+DTSTART:19700329T020000
+RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:+0200
+TZOFFSETTO:+0100
+TZNAME:CET
+DTSTART:19701025T030000
+RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+CREATED:20130902T150157Z
+LAST-MODIFIED:20130902T150158Z
+DTSTAMP:20130902T150158Z
+UID:event10
+SUMMARY:Event
+CATEGORIES:some_category1,another_category2
+ORGANIZER:mailto:unclesam@example.com
+ATTENDEE;ROLE=REQ-PARTICIPANT;PARTSTAT=TENTATIVE;CN=Jane Doe:MAILTO:janedoe@example.com
+ATTENDEE;ROLE=REQ-PARTICIPANT;DELEGATED-FROM="MAILTO:bob@host.com";PARTSTAT=ACCEPTED;CN=John Doe:MAILTO:johndoe@example.com
+DTSTART;TZID=Europe/Paris:20130901T180000
+DTEND;TZID=Europe/Paris:20130901T190000
+STATUS:CANCELLED
+END:VEVENT
+END:VCALENDAR

radicale/tests/test_base.py

@@ -25,6 +25,7 @@ import posixpath
 from typing import Any, Callable, ClassVar, Iterable, List, Optional, Tuple
 
 import defusedxml.ElementTree as DefusedET
+import vobject
 
 from radicale import storage, xmlutils
 from radicale.tests import RESPONSES, BaseTest
@@ -1360,10 +1361,45 @@ permissions: RrWw""")
 </C:calendar-query>""")
         assert len(responses) == 1
         response = responses[event_path]
-        assert not isinstance(response, int)
+        assert isinstance(response, dict)
         status, prop = response["D:getetag"]
         assert status == 200 and prop.text
 
+    def test_report_free_busy(self) -> None:
+        """Test free busy report on a few items"""
+        calendar_path = "/calendar.ics/"
+        self.mkcalendar(calendar_path)
+        for i in (1, 2, 10):
+            filename = "event{}.ics".format(i)
+            event = get_file_content(filename)
+            self.put(posixpath.join(calendar_path, filename), event)
+        code, responses = self.report(calendar_path, """\
+<?xml version="1.0" encoding="utf-8" ?>
+<C:free-busy-query xmlns:C="urn:ietf:params:xml:ns:caldav">
+    <C:time-range start="20130901T140000Z" end="20130908T220000Z"/>
+</C:free-busy-query>""", 200, is_xml=False)
+        for response in responses.values():
+            assert isinstance(response, vobject.base.Component)
+        assert len(responses) == 1
+        vcalendar = list(responses.values())[0]
+        assert isinstance(vcalendar, vobject.base.Component)
+        assert len(vcalendar.vfreebusy_list) == 3
+        types = {}
+        for vfb in vcalendar.vfreebusy_list:
+            fbtype_val = vfb.fbtype.value
+            if fbtype_val not in types:
+                types[fbtype_val] = 0
+            types[fbtype_val] += 1
+        assert types == {'BUSY': 2, 'FREE': 1}
+
+        # Test max_freebusy_occurrence limit
+        self.configure({"reporting": {"max_freebusy_occurrence": 1}})
+        code, responses = self.report(calendar_path, """\
+<?xml version="1.0" encoding="utf-8" ?>
+<C:free-busy-query xmlns:C="urn:ietf:params:xml:ns:caldav">
+    <C:time-range start="20130901T140000Z" end="20130908T220000Z"/>
+</C:free-busy-query>""", 400, is_xml=False)
+
     def _report_sync_token(
             self, calendar_path: str, sync_token: Optional[str] = None
             ) -> Tuple[str, RESPONSES]: