perf(calendar): pre-filter ICS data before parsing (#4168)

This adds a pre-filter that drops out-of-window non-recurring events
from the raw ICS data before `node-ical` parses it. Recurring events and
anything we cannot classify cheaply are kept - the existing logic
handles those as before.

Some calendars accumulate years of old events. Pre-filtering them
reduces the amount of work `node-ical` has to do and keeps event loop
lag lower. My benchmarks on a fast machine showed a small but measurable
speedup (roughly 10-20% with generated test data); on a Raspberry Pi it
should be significantly more noticeable.

For the implementation I ended up using
[`ics-filter`](https://github.com/runely/ics-filter), suggested by
@rejas. I had a custom version first, but ICS date string parsing has
enough complexity that it makes sense to delegate it to a package built
specifically for this. I also contributed a couple of fixes to
`ics-filter` along the way, and the maintainer was responsive and open
to improvements, so it seems like a good fit.

Solves #4103.

cspell.config.json

@@ -257,6 +257,7 @@
 		"pubdate",
 		"radokristof",
 		"rajniszp",
+		"RDATE",
 		"rebuilded",
 		"Reis",
 		"rejas",

defaultmodules/calendar/calendarfetcher.js

@@ -59,7 +59,13 @@ class CalendarFetcher {
 			}
 
 			const responseData = await response.text();
-			const parsed = await ical.async.parseICS(responseData);
+
+			const filteredData = await CalendarFetcherUtils.preFilterICS(responseData, {
+				includePastEvents: this.includePastEvents,
+				maximumNumberOfDays: this.maximumNumberOfDays
+			});
+
+			const parsed = await ical.async.parseICS(filteredData);
 
 			Log.debug(`Parsed iCal data from ${this.url} with ${Object.keys(parsed).length} entries.`);
 

defaultmodules/calendar/calendarfetcherutils.js

@@ -41,6 +41,36 @@ const CalendarFetcherUtils = {
 		return moment.tz.guess();
 	},
 
+	/**
+	 * Calculate the time window of events to keep, as [start, end].
+	 * Without includePastEvents the window starts now; otherwise it also
+	 * reaches maximumNumberOfDays into the past.
+	 * @param {object} config Needs includePastEvents (boolean) and maximumNumberOfDays (number).
+	 * @returns {[Date, Date]} The start and end of the window.
+	 */
+	calculateFilterWindow (config) {
+		const today = moment().startOf("day");
+		const start = config.includePastEvents
+			? today.clone().subtract(config.maximumNumberOfDays, "days").toDate()
+			: new Date();
+		const end = today.clone().add(config.maximumNumberOfDays, "days").toDate();
+		return [start, end];
+	},
+
+	/**
+	 * Drop ICS data outside the configured time window before it is parsed,
+	 * so that node-ical only has to process events we might actually show.
+	 * @param {string} rawICS The raw ICS text.
+	 * @param {object} config Needs includePastEvents (boolean) and maximumNumberOfDays (number).
+	 * @returns {Promise<string>} The filtered ICS text.
+	 */
+	async preFilterICS (rawICS, config) {
+		// ics-filter is ESM-only, so we import it dynamically from this CommonJS file.
+		const { icsFilter } = await import("ics-filter");
+		const [start, end] = CalendarFetcherUtils.calculateFilterWindow(config);
+		return icsFilter(rawICS, start, end);
+	},
+
 	/**
 	 * Filter the events from ical according to the given config
 	 * @param {object} data the calendar data from ical

package-lock.json

@@ -21,6 +21,7 @@
 				"helmet": "^8.2.0",
 				"html-to-text": "^10.0.0",
 				"iconv-lite": "^0.7.2",
+				"ics-filter": "^1.0.2",
 				"ipaddr.js": "^2.4.0",
 				"moment": "^2.30.1",
 				"moment-timezone": "^0.6.2",
@@ -5117,6 +5118,12 @@
 				"url": "https://opencollective.com/express"
 			}
 		},
+		"node_modules/ics-filter": {
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/ics-filter/-/ics-filter-1.0.2.tgz",
+			"integrity": "sha512-RwOyxGssy6PEpDx6OfPfqadkJQtIE29Huq6MFZ4PkZWyMtuyHRPAb4J6BpT9IzEwis6a4Lk5DAIsFrM2HBPP6A==",
+			"license": "MIT"
+		},
 		"node_modules/ignore": {
 			"version": "5.3.2",
 			"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",

package.json

@@ -93,6 +93,7 @@
 		"helmet": "^8.2.0",
 		"html-to-text": "^10.0.0",
 		"iconv-lite": "^0.7.2",
+		"ics-filter": "^1.0.2",
 		"ipaddr.js": "^2.4.0",
 		"moment": "^2.30.1",
 		"moment-timezone": "^0.6.2",

tests/unit/modules/default/calendar/calendar_fetcher_utils_spec.js

@@ -298,6 +298,29 @@ END:VCALENDAR`);
 		});
 	});
 
+	describe("calculateFilterWindow", () => {
+		it("ends maximumNumberOfDays after today's midnight", () => {
+			const [, end] = CalendarFetcherUtils.calculateFilterWindow({ includePastEvents: false, maximumNumberOfDays: 30 });
+
+			expect(end).toEqual(moment().startOf("day").add(30, "days").toDate());
+		});
+
+		it("starts now when includePastEvents is false", () => {
+			const before = Date.now();
+			const [start] = CalendarFetcherUtils.calculateFilterWindow({ includePastEvents: false, maximumNumberOfDays: 30 });
+			const after = Date.now();
+
+			expect(start.getTime()).toBeGreaterThanOrEqual(before);
+			expect(start.getTime()).toBeLessThanOrEqual(after);
+		});
+
+		it("starts maximumNumberOfDays before today's midnight when includePastEvents is true", () => {
+			const [start] = CalendarFetcherUtils.calculateFilterWindow({ includePastEvents: true, maximumNumberOfDays: 30 });
+
+			expect(start).toEqual(moment().startOf("day").subtract(30, "days").toDate());
+		});
+	});
+
 	describe("expandRecurringEvent", () => {
 		it("should extend end to end-of-day when event has no DTEND", () => {
 			// node-ical sets end === start when DTEND is absent; our code extends to endOf("day")