[Issue #1069] Improve performance of dashboard queries via new analytics db indexes and views

analytics/src/analytics/integrations/etldb/main.py

@@ -133,6 +133,30 @@ def sync_data(
     m = f"issue row(s) processed: {len(issue_map)}"
     logger.info(m)
 
+    # refresh deliverable/epic summary views used by burndown queries --
+    # see migration 0015; keeps them no more than one sync cycle stale
+    refresh_materialized_views(db)
+
+
+def refresh_materialized_views(db: EtlDb) -> None:
+    """
+    Refresh the deliverable/epic summary views used by burndown queries.
+
+    CONCURRENTLY avoids locking out readers while the refresh runs, at the cost of
+    requiring a unique index on each view (see migration 0015). Order matters:
+    mv_deliverable_daily_burndown reads from mv_latest_epic_deliverable_map, so the
+    latter must be refreshed first or the former would recompute against a stale
+    epic-to-deliverable mapping.
+    """
+    cursor = db.connection()
+    for view in (
+        "mv_deliverables_per_quad",
+        "mv_latest_epic_deliverable_map",
+        "mv_deliverable_daily_burndown",
+    ):
+        cursor.execute(text(f"REFRESH MATERIALIZED VIEW CONCURRENTLY {view}"))
+    db.commit(cursor)
+
 
 def sync_deliverables(
     db: EtlDb,

analytics/src/analytics/integrations/etldb/migrations/versions/0014_add_burndown_query_indexes.sql

@@ -0,0 +1,13 @@
+-- gh_issue.parent_issue_ghid drives the recursive epic->issue tree walk used
+-- by the Deliverable_Burndowns/Deliverable_Burnup/Deliverable_Data queries;
+-- it previously had no index at all, forcing a full table scan per
+-- recursion level per query.
+CREATE INDEX IF NOT EXISTS gh_issue_i2 ON gh_issue(parent_issue_ghid);
+
+-- gh_deliverable_quad_map/gh_epic_deliverable_map are day-keyed SCD tables;
+-- "latest row per deliverable/epic" is resolved via a ROW_NUMBER()/DISTINCT
+-- ON ordered (id, d_effective DESC), but the existing indexes on these
+-- tables are ascending on d_effective, so that ordering can't be served
+-- from an index and falls back to a full sort.
+CREATE INDEX IF NOT EXISTS gh_dqm_i2 ON gh_deliverable_quad_map(deliverable_id, d_effective DESC);
+CREATE INDEX IF NOT EXISTS gh_edm_i2 ON gh_epic_deliverable_map(epic_id, d_effective DESC);

analytics/src/analytics/integrations/etldb/migrations/versions/0015_add_deliverable_epic_materialized_views.sql

@@ -0,0 +1,96 @@
+-- Precomputed per-quad deliverable ranking, current-state and independent
+-- of any reporting period. Lets deliverable-ranking queries read a ranked
+-- deliverable list directly instead of recomputing the window
+-- function/anti-join over the full history tables on every query.
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS mv_deliverables_per_quad AS
+WITH quad AS (
+  SELECT id AS quad_id, name AS quad_name, ghid AS quad_ghid, start_date, end_date
+  FROM gh_quad
+),
+quad_deliverable_map AS (
+  SELECT *
+  FROM (
+    SELECT quad_id, deliverable_id, d_effective,
+           ROW_NUMBER() OVER (PARTITION BY deliverable_id ORDER BY d_effective DESC) AS ranked_order
+    FROM gh_deliverable_quad_map
+  ) history
+  WHERE history.ranked_order = 1
+),
+all_deliverables AS (
+  SELECT d.id AS deliverable_id, d.title AS deliverable_title, d.ghid AS deliverable_ghid
+  FROM gh_deliverable d
+  WHERE NOT EXISTS (SELECT 1 FROM gh_epic e WHERE e.ghid = d.ghid AND e.t_modified > d.t_modified)
+    AND NOT EXISTS (SELECT 1 FROM gh_issue i WHERE i.ghid = d.ghid AND i.t_modified > d.t_modified)
+)
+SELECT q.quad_id, q.quad_name, q.quad_ghid,
+       d.deliverable_id, d.deliverable_title, d.deliverable_ghid,
+       qdm.d_effective,
+       ROW_NUMBER() OVER (PARTITION BY q.quad_id ORDER BY d.deliverable_title) AS ranked_order
+FROM quad q
+JOIN quad_deliverable_map qdm ON q.quad_id = qdm.quad_id
+JOIN all_deliverables d ON qdm.deliverable_id = d.deliverable_id;
+
+-- Each deliverable is mapped to exactly one (its latest) quad, so
+-- deliverable_id alone is unique -- required for REFRESH ... CONCURRENTLY.
+CREATE UNIQUE INDEX IF NOT EXISTS mv_dpq_i1 ON mv_deliverables_per_quad (deliverable_id);
+
+-- Precomputed latest epic-to-deliverable mapping, one row per epic holding
+-- its most recent mapping by d_effective. Lets burndowns queries join
+-- directly instead of recomputing a DISTINCT ON over the full mapping
+-- history on every query.
+CREATE MATERIALIZED VIEW IF NOT EXISTS mv_latest_epic_deliverable_map AS
+SELECT DISTINCT ON (edm.epic_id) edm.epic_id, e.ghid AS epic_ghid, edm.deliverable_id, edm.d_effective
+FROM gh_epic_deliverable_map edm
+JOIN gh_epic e ON edm.epic_id = e.id
+ORDER BY edm.epic_id, edm.d_effective DESC;
+
+CREATE UNIQUE INDEX IF NOT EXISTS mv_ledm_i1 ON mv_latest_epic_deliverable_map (epic_id);
+
+-- Precomputed deliverable-to-issue membership (epic tree walk plus direct
+-- children) joined against gh_issue_history and pre-aggregated by
+-- (deliverable_id, day). Lets burndowns queries employ a single indexed
+-- range scan instead of a recursive tree walk and full history join.
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS mv_deliverable_daily_burndown AS
+WITH RECURSIVE
+epics_in_deliverable AS (
+  SELECT lem.epic_id, lem.epic_ghid, lem.deliverable_id
+  FROM mv_latest_epic_deliverable_map lem
+),
+epic_issue_tree AS (
+  SELECT e.deliverable_id, i.id AS issue_id, i.ghid AS issue_ghid
+  FROM epics_in_deliverable e
+  JOIN gh_issue i ON i.parent_issue_ghid = e.epic_ghid
+  UNION ALL
+  SELECT eit.deliverable_id, i.id AS issue_id, i.ghid AS issue_ghid
+  FROM gh_issue i
+  JOIN epic_issue_tree eit ON i.parent_issue_ghid = eit.issue_ghid
+),
+direct_issues AS (
+  SELECT d.id AS deliverable_id, i.id AS issue_id
+  FROM gh_deliverable d
+  JOIN gh_issue i ON i.parent_issue_ghid = d.ghid
+  WHERE NOT EXISTS (
+    SELECT 1
+    FROM mv_latest_epic_deliverable_map lem
+    JOIN gh_epic ep ON lem.epic_id = ep.id
+    WHERE ep.ghid = i.ghid
+  )
+),
+deliverable_issue_map AS (
+  SELECT deliverable_id, issue_id FROM epic_issue_tree
+  UNION ALL
+  SELECT deliverable_id, issue_id FROM direct_issues
+)
+SELECT dim.deliverable_id,
+       h.d_effective AS issue_day,
+       COUNT(DISTINCT h.issue_id) AS total_issues_opened,
+       COUNT(DISTINCT h.issue_id) FILTER (WHERE h.is_closed::BOOLEAN) AS total_issues_closed,
+       COALESCE(SUM(h.points), 0) AS total_points_opened,
+       COALESCE(SUM(h.points) FILTER (WHERE h.is_closed::BOOLEAN), 0) AS total_points_closed
+FROM deliverable_issue_map dim
+JOIN gh_issue_history h ON h.issue_id = dim.issue_id
+GROUP BY dim.deliverable_id, h.d_effective;
+
+CREATE UNIQUE INDEX IF NOT EXISTS mv_ddb_i1 ON mv_deliverable_daily_burndown (deliverable_id, issue_day);

analytics/src/analytics/integrations/metabase/sql/67-Sprint_Metrics/155-Sprint_Burnup_by_points.sql

@@ -10,36 +10,35 @@ WITH -- get project_id
    FROM project_data,
         gh_sprint
    WHERE gh_sprint.project_id = project_data.project_id
-     AND {{sprint_name}}), -- calculate points opened in the sprint
- opened AS
+     AND {{sprint_name}}), -- calculate total sprint scope (points tracked) by day
+ SCOPE AS
   (SELECT gh_issue_history.d_effective AS DAY,
-          sum(gh_issue_history.points) AS total_opened
+          sum(gh_issue_history.points) AS total_scope
    FROM gh_issue_history,
         sprint_data
    WHERE gh_issue_history.sprint_id = sprint_data.sprint_id
      AND (gh_issue_history.d_effective >= sprint_data.sprint_start_date
           AND gh_issue_history.d_effective <= sprint_data.sprint_end_date)
    GROUP BY DAY
-   ORDER BY DAY), -- calculate points closed in the sprint
- closed AS
+   ORDER BY DAY), -- calculate points completed in the sprint by day
+ completed AS
   (SELECT gh_issue_history.d_effective AS DAY,
-          sum(gh_issue_history.points) AS total_closed
+          sum(gh_issue_history.points) AS total_completed
    FROM gh_issue_history,
         sprint_data
    WHERE gh_issue_history.sprint_id = sprint_data.sprint_id
      AND gh_issue_history.is_closed = 1
      AND (gh_issue_history.d_effective >= sprint_data.sprint_start_date
           AND gh_issue_history.d_effective <= sprint_data.sprint_end_date)
    GROUP BY DAY
-   ORDER BY DAY), -- aggregate points opened and closed by day
+   ORDER BY DAY), -- aggregate scope and completed work by day (both climb toward total scope)
  totals AS
-  (SELECT opened.day AS DAY,
-          opened.total_opened,
-          closed.total_closed,
-          opened.total_opened - closed.total_closed AS total_remaining
-   FROM opened,
-        closed
-   WHERE opened.day = closed.day
+  (SELECT scope.day AS DAY,
+          scope.total_scope,
+          completed.total_completed
+   FROM SCOPE,
+        completed
+   WHERE scope.day = completed.day
    ORDER BY DAY)
 SELECT *
-FROM totals
+FROM totals

analytics/src/analytics/integrations/metabase/sql/67-Sprint_Metrics/156-Sprint_Burnup_by_issues.sql

@@ -10,36 +10,35 @@ WITH -- get project_id
    FROM project_data,
         gh_sprint
    WHERE gh_sprint.project_id = project_data.project_id
-     AND {{sprint_name}}), -- calculate issues opened in the sprint identified by sprint_id
- opened AS
+     AND {{sprint_name}}), -- calculate total sprint scope (issues tracked) by day
+ SCOPE AS
   (SELECT gh_issue_history.d_effective AS DAY,
-          count(*) AS total_opened
+          count(*) AS total_scope
    FROM gh_issue_history,
         sprint_data
    WHERE gh_issue_history.sprint_id = sprint_data.sprint_id
      AND (gh_issue_history.d_effective >= sprint_data.sprint_start_date
           AND gh_issue_history.d_effective <= sprint_data.sprint_end_date)
    GROUP BY DAY
-   ORDER BY DAY), -- calculate issues closed in the sprint
- closed AS
+   ORDER BY DAY), -- calculate issues completed in the sprint by day
+ completed AS
   (SELECT gh_issue_history.d_effective AS DAY,
-          count(*) AS total_closed
+          count(*) AS total_completed
    FROM gh_issue_history,
         sprint_data
    WHERE gh_issue_history.sprint_id = sprint_data.sprint_id
      AND gh_issue_history.is_closed = 1
      AND (gh_issue_history.d_effective >= sprint_data.sprint_start_date
           AND gh_issue_history.d_effective <= sprint_data.sprint_end_date)
    GROUP BY DAY
-   ORDER BY DAY), -- aggregate issues opened and closed by day
+   ORDER BY DAY), -- aggregate scope and completed work by day (both climb toward total scope)
  totals AS
-  (SELECT opened.day AS DAY,
-          opened.total_opened,
-          closed.total_closed,
-          opened.total_opened - closed.total_closed AS total_remaining
-   FROM opened,
-        closed
-   WHERE opened.day = closed.day
+  (SELECT scope.day AS DAY,
+          scope.total_scope,
+          completed.total_completed
+   FROM SCOPE,
+        completed
+   WHERE scope.day = completed.day
    ORDER BY DAY)
 SELECT *
-FROM totals
+FROM totals

analytics/src/analytics/integrations/metabase/sql/69-Delivery_Metrics/212-Deliverable_Burnup_Issue_Count.sql

@@ -65,33 +65,32 @@ WITH RECURSIVE -- 1. Get reporting period dynamically
   (SELECT *
    FROM epic_issue_tree
    UNION ALL SELECT *
-   FROM direct_issues), -- 9. Calculate issues opened in the given time period
- opened AS
+   FROM direct_issues), -- 9. Calculate total issue scope (issues tracked) in the given time period
+ scope AS
   (SELECT h.d_effective AS issue_day,
-          COUNT(DISTINCT h.issue_id) AS total_opened
+          COUNT(DISTINCT h.issue_id) AS total_scope
    FROM gh_issue_history h
    JOIN combined_issues ci ON h.issue_id = ci.issue_id
    CROSS JOIN reporting_period
    WHERE h.d_effective BETWEEN reporting_period.start_date AND reporting_period.end_date
    GROUP BY h.d_effective
-   ORDER BY h.d_effective), -- 10. Calculate issues closed in the given time period
- closed AS
+   ORDER BY h.d_effective), -- 10. Calculate issues completed in the given time period
+ completed AS
   (SELECT h.d_effective AS issue_day,
-          COUNT(DISTINCT h.issue_id) AS total_closed
+          COUNT(DISTINCT h.issue_id) AS total_completed
    FROM gh_issue_history h
    JOIN combined_issues ci ON h.issue_id = ci.issue_id
    CROSS JOIN reporting_period
    WHERE h.is_closed::BOOLEAN = TRUE
      AND h.d_effective BETWEEN reporting_period.start_date AND reporting_period.end_date
    GROUP BY h.d_effective
-   ORDER BY h.d_effective), -- 11. Aggregate issues opened and closed by day
+   ORDER BY h.d_effective), -- 11. Aggregate scope and completed work by day (both climb toward total scope)
  totals AS
-  (SELECT COALESCE(o.issue_day, c.issue_day) AS issue_day,
-          COALESCE(o.total_opened, 0) AS total_opened,
-          COALESCE(c.total_closed, 0) AS total_closed,
-          COALESCE(o.total_opened, 0) - COALESCE(c.total_closed, 0) AS total_remaining
-   FROM opened o
-   FULL OUTER JOIN closed c ON o.issue_day = c.issue_day
+  (SELECT COALESCE(s.issue_day, c.issue_day) AS issue_day,
+          COALESCE(s.total_scope, 0) AS total_scope,
+          COALESCE(c.total_completed, 0) AS total_completed
+   FROM scope s
+   FULL OUTER JOIN completed c ON s.issue_day = c.issue_day
    ORDER BY issue_day)
 SELECT *
 FROM totals;

analytics/src/analytics/integrations/metabase/sql/69-Delivery_Metrics/213-Deliverable_Burnup_Points.sql

@@ -65,33 +65,32 @@ WITH RECURSIVE -- 1. Get reporting period dynamically
   (SELECT *
    FROM epic_issue_tree
    UNION ALL SELECT *
-   FROM direct_issues), -- 9. Calculate points opened in the given time period
- opened AS
+   FROM direct_issues), -- 9. Calculate total points scope (points tracked) in the given time period
+ scope AS
   (SELECT h.d_effective AS issue_day,
-          SUM(h.points) AS total_opened
+          SUM(h.points) AS total_scope
    FROM gh_issue_history h
    JOIN combined_issues ci ON h.issue_id = ci.issue_id
    CROSS JOIN reporting_period
    WHERE h.d_effective BETWEEN reporting_period.start_date AND reporting_period.end_date
    GROUP BY h.d_effective
-   ORDER BY h.d_effective), -- 10. Calculate points closed in the given time period
- closed AS
+   ORDER BY h.d_effective), -- 10. Calculate points completed in the given time period
+ completed AS
   (SELECT h.d_effective AS issue_day,
-          SUM(h.points) AS total_closed
+          SUM(h.points) AS total_completed
    FROM gh_issue_history h
    JOIN combined_issues ci ON h.issue_id = ci.issue_id
    CROSS JOIN reporting_period
    WHERE h.is_closed::BOOLEAN = TRUE
      AND h.d_effective BETWEEN reporting_period.start_date AND reporting_period.end_date
    GROUP BY h.d_effective
-   ORDER BY h.d_effective), -- 11. Aggregate points opened and closed by day
+   ORDER BY h.d_effective), -- 11. Aggregate scope and completed work by day (both climb toward total scope)
  totals AS
-  (SELECT COALESCE(o.issue_day, c.issue_day) AS issue_day,
-          COALESCE(o.total_opened, 0) AS total_opened,
-          COALESCE(c.total_closed, 0) AS total_closed,
-          COALESCE(o.total_opened, 0) - COALESCE(c.total_closed, 0) AS total_remaining
-   FROM opened o
-   FULL OUTER JOIN closed c ON o.issue_day = c.issue_day
+  (SELECT COALESCE(s.issue_day, c.issue_day) AS issue_day,
+          COALESCE(s.total_scope, 0) AS total_scope,
+          COALESCE(c.total_completed, 0) AS total_completed
+   FROM scope s
+   FULL OUTER JOIN completed c ON s.issue_day = c.issue_day
    ORDER BY issue_day)
 SELECT *
 FROM totals;