fix(chart) :: align stacked series on their X values

CHANGELOG.md

@@ -7,6 +7,7 @@
  - `sqlpage.send_mail` now supports rich email bodies. Use `body_html` for a caller-provided HTML alternative, or `body_md` to render Markdown as HTML. Messages retain a plain-text alternative; `body` may be omitted when `body_md` is used, and `body_md` and `body_html` cannot be combined.
  - Form `options_source` URLs now preserve existing query parameters when adding the dynamic `search` parameter.
  - Map coordinates that are not a pair of numbers, like a latitude with no longitude, are now reported in the browser console and skipped, instead of breaking the whole map.
+ - Stacked charts now stack their series by `x` value instead of by point order, which used to give wrong totals when a series was missing a point.
  - `column` charts now display vertical bars instead of nothing at all.
  - `stacked` is now ignored on chart types that cannot stack, instead of displaying an empty chart.
  - Screen readers now announce the title of the modal component instead of an unnamed dialog.

examples/official-site/sqlpage/migrations/01_documentation.sql

@@ -664,7 +664,7 @@ INSERT INTO parameter(component, name, description, type, top_level, optional) S
     ('marker', 'Marker size', 'REAL', TRUE, TRUE),
     ('labels', 'Whether to show the data labels on the chart or not.', 'BOOLEAN', TRUE, TRUE),
     ('color', 'The name of a color in which to display the chart. If there are multiple series in the chart, this parameter can be repeated multiple times.', 'COLOR', TRUE, TRUE),
-    ('stacked', 'Whether to cumulate values from different series. Supported by the "line", "area" and "bar" chart types, and ignored by the others.', 'BOOLEAN', TRUE, TRUE),
+    ('stacked', 'Whether to cumulate values from different series. Supported by the "line", "area" and "bar" chart types, and ignored by the others. Series are aligned on their x values, and a series that has no value for a given x counts as zero there.', 'BOOLEAN', TRUE, TRUE),
     ('toolbar', 'Whether to display a toolbar at the top right of the chart, that offers downloading the data as CSV.', 'BOOLEAN', TRUE, TRUE),
     ('show_legend', 'Whether to display the legend listing all chart series. Defaults to true.', 'BOOLEAN', TRUE, TRUE),
     ('logarithmic', 'Display the y-axis in logarithmic scale.', 'BOOLEAN', TRUE, TRUE),
@@ -717,6 +717,20 @@ INSERT INTO example(component, description, properties) VALUES
     '{"series": "Marketing", "x": 2022, "value": 15}, '||
     '{"series": "Human resources", "x": 2021, "value": 30}, '||
     '{"series": "Human resources", "x": 2022, "value": 55}]')),
+    ('chart', 'A stacked area chart, showing how each series contributes to a total.
+The `stacked` property also works with the `line` and `bar` chart types.
+
+Series are aligned on their `x` values, and a series that has no value for a given `x` counts as zero there:
+below, the graphics card draws no power outside of the render.
+If a missing value does not mean zero in your data, make all the series share the same `x` values,
+for instance by rounding timestamps to a common interval.',
+    json('[{"component":"chart", "title": "Power draw", "type": "area", "stacked": true, "time": true, "ytitle": "watts", "color": ["blue", "teal"], "marker": 4}, '||
+    '{"series": "CPU", "x": "2024-03-01T10:00:00Z", "value": 45}, '||
+    '{"series": "CPU", "x": "2024-03-01T10:15:00Z", "value": 52}, '||
+    '{"series": "CPU", "x": "2024-03-01T10:30:00Z", "value": 48}, '||
+    '{"series": "CPU", "x": "2024-03-01T10:45:00Z", "value": 44}, '||
+    '{"series": "GPU", "x": "2024-03-01T10:15:00Z", "value": 120}, '||
+    '{"series": "GPU", "x": "2024-03-01T10:30:00Z", "value": 140}]')),
     ('chart', 'A line chart with multiple series. One of the most common types of charts, often used to show trends over time.
 Also demonstrates the use of the `toolbar` attribute to allow the user to download the graph as an image or the data as a CSV file.', 
     json('[{"component":"chart", "title": "Revenue", "ymin": 0, "toolbar": true},

package.json

@@ -2,7 +2,7 @@
   "name": "sqlpage",
   "version": "1.0.0",
   "scripts": {
-    "test": "biome check .",
+    "test": "biome check . && node --test \"tests/js/**/*.spec.ts\"",
     "format": "biome format --write .",
     "fix": "biome check --fix --unsafe ."
   },

sqlpage/apexcharts.js

@@ -39,61 +39,57 @@ sqlpage_chart = (() => {
   const STACKABLE_CHART_TYPES = ["line", "area", "bar"];
   const APEXCHARTS_TYPE_ALIASES = { column: "bar" };
 
-  /** @typedef { { [name:string]: {data:{x:number|string|Date,y:number}[], name:string} } } Series */
+  /** @typedef {number|string|Date} XValue */
+  /** @typedef { {name:string, data:{x:XValue,y:number|null,z?:number}[]} } ChartSeries */
+  /** @typedef { { [name:string]: ChartSeries } } Series */
+
+  /** @param {XValue} x @returns {number|string} equal x values share a key */
+  const x_key = (x) => (x instanceof Date ? x.getTime() : x);
 
   /**
-   * Aligns series data points by their x-axis categories, ensuring all series have data points
-   * for each unique category. Missing values are filled with zeros.
-   * Categories are ordered by their name.
-   *
-   * @example
-   * // Input series:
-   * const series = [
-   *   { name: "A", data: [{x: "X2", y: 10}, {x: "X3", y: 30}] },
-   *   { name: "B", data: [{x: "X1", y: 25}, {x: "X2", y: 20}] }
-   * ];
-   *
-   * // Output after align_categories (orderedCategories will be ["X1","X2", "X3"]):
-   * // [
-   * //   { name: "A", data: [{x: "X1", y: 0}, {x: "X2", y: 10}, {x: "X3", y: 30}] },
-   * //   { name: "B", data: [{x: "X1", y: 25}, {x: "X2", y: 20}, {x: "X3", y: 0}] }
-   * // ]
-   *
-   * @param {(Series[string])[]} series - Array of series objects, each containing name and data points
-   * @returns {Series[string][]} Aligned series with consistent categories across all series
+   * @param {ChartSeries[]} series
+   * @returns {XValue[]} every x the series hold, in their own order where they
+   * agree and in ascending order where they diverge
    */
-  function align_categories(series) {
-    const categoriesSet = new Set();
-    const pointers = series.map((_) => 0); // Index of current data point in each series
-    const x_at = (series_idx) =>
-      series[series_idx].data[pointers[series_idx]].x;
-    const series_idxs = series.flatMap((s, i) => (s.data.length ? i : []));
-    while (series_idxs.length > 0) {
-      let idx_of_xmin = series_idxs[0];
-      for (const series_idx of series_idxs) {
-        if (x_at(series_idx) < x_at(idx_of_xmin)) idx_of_xmin = series_idx;
-      }
-
-      const new_category = x_at(idx_of_xmin);
-      if (!categoriesSet.has(new_category)) categoriesSet.add(new_category);
-      pointers[idx_of_xmin]++;
-      if (pointers[idx_of_xmin] >= series[idx_of_xmin].data.length) {
-        series_idxs.splice(series_idxs.indexOf(idx_of_xmin), 1);
-      }
+  function merged_x_values(series) {
+    const unread = series.map(({ data }) => data.map(({ x }) => x));
+    const merged = new Map();
+    while (unread.some((xs) => xs.length > 0)) {
+      const with_lowest_x = unread
+        .filter((xs) => xs.length > 0)
+        .reduce((a, b) => (b[0] < a[0] ? b : a));
+      const x = with_lowest_x.shift();
+      merged.set(x_key(x), x);
     }
-    // Create a map of category -> value for each series and rebuild
-    return series.map((s) => {
-      const valueMap = new Map(s.data.map((point) => [point.x, point.y]));
+    return [...merged.values()];
+  }
+
+  /**
+   * ApexCharts pairs points across series by index rather than by x, so a
+   * series that skips an x stacks onto the wrong one. Give every series the
+   * same x values, counting an x it never measured as zero.
+   *
+   * @param {ChartSeries[]} series
+   * @returns {ChartSeries[]}
+   */
+  function align_series(series) {
+    const all_x = merged_x_values(series);
+    return series.map(({ name, data }) => {
+      const by_x = new Map(data.map((point) => [x_key(point.x), point]));
       return {
-        name: s.name,
-        data: Array.from(categoriesSet, (category) => ({
-          x: category,
-          y: valueMap.get(category) || 0,
-        })),
+        name,
+        data: all_x.map((x) => {
+          const point = by_x.get(x_key(x));
+          return { ...point, x, y: point?.y || 0 };
+        }),
       };
     });
   }
 
+  // The unit tests load this file as a CommonJS module; browsers have no `module`.
+  if (typeof module !== "undefined")
+    module.exports = { align_series, merged_x_values };
+
   /** @param {HTMLElement} c */
   function build_sqlpage_chart(c) {
     const [data_element] = c.getElementsByTagName("data");
@@ -138,8 +134,11 @@ sqlpage_chart = (() => {
     if (chart_type === "pie") {
       labels = data.points.map(([name, x, _y]) => x || name);
       series = data.points.map(([_name, _x, y]) => Number.parseFloat(y));
-    } else if (categories && chart_type === "bar" && series.length > 1)
-      series = align_categories(series);
+    } else if (
+      series.length > 1 &&
+      (is_stacked || (categories && chart_type === "bar"))
+    )
+      series = align_series(series);
 
     const options = {
       chart: {

tests/end-to-end/chart-component.spec.ts

@@ -2,10 +2,17 @@ import { expect, type Page, test } from "@playwright/test";
 
 const BASE = process.env.SQLPAGE_TEST_BASE ?? "http://localhost:8080/";
 
+type ChartPoint = { x: string | number | Date; y: number | null };
+
 declare global {
   interface Window {
     charts?: {
-      w: { config: { chart: { type: string; stacked: boolean } } };
+      w: {
+        config: {
+          chart: { type: string; stacked: boolean };
+          series: { name: string; data: ChartPoint[] }[];
+        };
+      };
     }[];
   }
   function sqlpage_chart(): void;
@@ -24,6 +31,46 @@ const TASKS_OVER_TIME: Row[] = [
   ["Build", "Bob", ["2024-03-04", "2024-03-09"]],
 ];
 
+const CPU_AT_EVERY_MINUTE: Row[] = [
+  ["CPU", "2024-01-01T00:00:00Z", 10],
+  ["CPU", "2024-01-01T00:01:00Z", 20],
+  ["CPU", "2024-01-01T00:02:00Z", 30],
+  ["CPU", "2024-01-01T00:03:00Z", 40],
+];
+
+const GPU_ONLY_ONCE_THE_RENDER_STARTED: Row[] = [
+  ["GPU", "2024-01-01T00:01:00Z", 50],
+  ["GPU", "2024-01-01T00:02:00Z", 50],
+  ["GPU", "2024-01-01T00:03:00Z", 50],
+];
+
+const A_IN_EVERY_QUARTER: Row[] = [
+  ["A", "Q1", 1],
+  ["A", "Q2", 2],
+  ["A", "Q3", 3],
+];
+
+const B_MISSING_THE_FIRST_QUARTER: Row[] = [
+  ["B", "Q2", 20],
+  ["B", "Q3", 30],
+];
+
+const A_QUARTERS_OUT_OF_ORDER: Row[] = [
+  ["A", "Q3", 3],
+  ["A", "Q1", 1],
+  ["A", "Q2", 2],
+];
+
+const A_FROM_THE_SECOND_CATEGORY: Row[] = [
+  ["A", "X2", 10],
+  ["A", "X3", 30],
+];
+
+const B_UNTIL_THE_SECOND_CATEGORY: Row[] = [
+  ["B", "X1", 25],
+  ["B", "X2", 20],
+];
+
 async function renderChart(
   page: Page,
   chart: Record<string, unknown>,
@@ -52,6 +99,21 @@ async function renderChart(
       console.error = reportError;
 
       const rendered = window.charts?.[before];
+      const series = (rendered?.w.config.series ?? []).map((s) => ({
+        name: s.name,
+        points: s.data.map((p) => [
+          p.x instanceof Date ? p.x.toISOString() : p.x,
+          p.y,
+        ]),
+      }));
+      const drawnPerSeries = series.map(({ name }) => ({
+        name,
+        heights: [
+          ...container.querySelectorAll<SVGGraphicsElement>(
+            `.apexcharts-series[seriesName='${name}'] .apexcharts-marker`,
+          ),
+        ].map((m) => Math.round(m.getBBox().y)),
+      }));
       const shapes = [
         ...container.querySelectorAll<SVGGraphicsElement>(
           ".apexcharts-bar-area, .apexcharts-rangebar-area",
@@ -65,6 +127,8 @@ async function renderChart(
         failures,
         type: rendered?.w.config.chart.type ?? null,
         stacked: rendered?.w.config.chart.stacked ?? null,
+        series,
+        drawnPerSeries,
         shapes,
       };
     },
@@ -88,6 +152,106 @@ test("draws a column chart as a vertical bar chart", async ({ page }) => {
   expect(new Set(chart.shapes.map((s) => s.height)).size).toBe(3);
 });
 
+test("gives a stacked series a zero at every x it did not measure", async ({
+  page,
+}) => {
+  const chart = await renderChart(
+    page,
+    { type: "area", stacked: true, time: true },
+    [...CPU_AT_EVERY_MINUTE, ...GPU_ONLY_ONCE_THE_RENDER_STARTED],
+  );
+
+  expect(chart.failures).toEqual([]);
+  expect(chart.series.map((s) => s.name)).toEqual(["CPU", "GPU"]);
+  expect(chart.series[1].points).toEqual([
+    ["2024-01-01T00:00:00.000Z", 0],
+    ["2024-01-01T00:01:00.000Z", 50],
+    ["2024-01-01T00:02:00.000Z", 50],
+    ["2024-01-01T00:03:00.000Z", 50],
+  ]);
+});
+
+test("stacks a series above the one it shares an x with", async ({ page }) => {
+  const chart = await renderChart(
+    page,
+    { type: "area", stacked: true, time: true },
+    [...CPU_AT_EVERY_MINUTE, ...GPU_ONLY_ONCE_THE_RENDER_STARTED],
+  );
+  const [cpu, gpu] = chart.drawnPerSeries;
+
+  expect(gpu.heights).toHaveLength(4);
+  expect(gpu.heights[0]).toBe(cpu.heights[0]);
+  expect(gpu.heights[1]).toBeLessThan(cpu.heights[1]);
+});
+
+test("keeps a lone series in the order the query returned it (#930)", async ({
+  page,
+}) => {
+  const chart = await renderChart(
+    page,
+    { type: "bar" },
+    A_QUARTERS_OUT_OF_ORDER,
+  );
+
+  expect(chart.failures).toEqual([]);
+  expect(chart.series[0].points).toEqual([
+    ["Q3", 3],
+    ["Q1", 1],
+    ["Q2", 2],
+  ]);
+});
+
+test("orders by name the categories two bar series do not share (#951)", async ({
+  page,
+}) => {
+  const chart = await renderChart(page, { type: "bar" }, [
+    ...A_FROM_THE_SECOND_CATEGORY,
+    ...B_UNTIL_THE_SECOND_CATEGORY,
+  ]);
+
+  expect(chart.failures).toEqual([]);
+  expect(chart.series[0].points).toEqual([
+    ["X1", 0],
+    ["X2", 10],
+    ["X3", 30],
+  ]);
+  expect(chart.series[1].points).toEqual([
+    ["X1", 25],
+    ["X2", 20],
+    ["X3", 0],
+  ]);
+});
+
+test("leaves the points of a chart that does not stack alone", async ({
+  page,
+}) => {
+  const chart = await renderChart(page, { type: "area", time: true }, [
+    ...CPU_AT_EVERY_MINUTE,
+    ...GPU_ONLY_ONCE_THE_RENDER_STARTED,
+  ]);
+
+  expect(chart.failures).toEqual([]);
+  expect(chart.series[1].points).toEqual([
+    ["2024-01-01T00:01:00.000Z", 50],
+    ["2024-01-01T00:02:00.000Z", 50],
+    ["2024-01-01T00:03:00.000Z", 50],
+  ]);
+});
+
+test("stacks a bar series on the categories it skipped", async ({ page }) => {
+  const chart = await renderChart(page, { type: "bar", stacked: true }, [
+    ...A_IN_EVERY_QUARTER,
+    ...B_MISSING_THE_FIRST_QUARTER,
+  ]);
+
+  expect(chart.failures).toEqual([]);
+  expect(chart.series[1].points).toEqual([
+    ["Q1", 0],
+    ["Q2", 20],
+    ["Q3", 30],
+  ]);
+});
+
 test("draws a rangeBar chart that asks to be stacked", async ({ page }) => {
   const chart = await renderChart(
     page,