diff --git a/analytics/src/analytics/integrations/etldb/main.py b/analytics/src/analytics/integrations/etldb/main.py index c149861526..995e2e86db 100644 --- a/analytics/src/analytics/integrations/etldb/main.py +++ b/analytics/src/analytics/integrations/etldb/main.py @@ -133,6 +133,30 @@ def sync_data( m = f"issue row(s) processed: {len(issue_map)}" logger.info(m) + # refresh deliverable/epic summary views used by burndown queries -- + # see migration 0015; keeps them no more than one sync cycle stale + refresh_materialized_views(db) + + +def refresh_materialized_views(db: EtlDb) -> None: + """ + Refresh the deliverable/epic summary views used by burndown queries. + + CONCURRENTLY avoids locking out readers while the refresh runs, at the cost of + requiring a unique index on each view (see migration 0015). Order matters: + mv_deliverable_daily_burndown reads from mv_latest_epic_deliverable_map, so the + latter must be refreshed first or the former would recompute against a stale + epic-to-deliverable mapping. + """ + cursor = db.connection() + for view in ( + "mv_deliverables_per_quad", + "mv_latest_epic_deliverable_map", + "mv_deliverable_daily_burndown", + ): + cursor.execute(text(f"REFRESH MATERIALIZED VIEW CONCURRENTLY {view}")) + db.commit(cursor) + def sync_deliverables( db: EtlDb, diff --git a/analytics/src/analytics/integrations/etldb/migrations/versions/0014_add_burndown_query_indexes.sql b/analytics/src/analytics/integrations/etldb/migrations/versions/0014_add_burndown_query_indexes.sql new file mode 100644 index 0000000000..a96c48c35d --- /dev/null +++ b/analytics/src/analytics/integrations/etldb/migrations/versions/0014_add_burndown_query_indexes.sql @@ -0,0 +1,13 @@ +-- gh_issue.parent_issue_ghid drives the recursive epic->issue tree walk used +-- by the Deliverable_Burndowns/Deliverable_Burnup/Deliverable_Data queries; +-- it previously had no index at all, forcing a full table scan per +-- recursion level per query. +CREATE INDEX IF NOT EXISTS gh_issue_i2 ON gh_issue(parent_issue_ghid); + +-- gh_deliverable_quad_map/gh_epic_deliverable_map are day-keyed SCD tables; +-- "latest row per deliverable/epic" is resolved via a ROW_NUMBER()/DISTINCT +-- ON ordered (id, d_effective DESC), but the existing indexes on these +-- tables are ascending on d_effective, so that ordering can't be served +-- from an index and falls back to a full sort. +CREATE INDEX IF NOT EXISTS gh_dqm_i2 ON gh_deliverable_quad_map(deliverable_id, d_effective DESC); +CREATE INDEX IF NOT EXISTS gh_edm_i2 ON gh_epic_deliverable_map(epic_id, d_effective DESC); diff --git a/analytics/src/analytics/integrations/etldb/migrations/versions/0015_add_deliverable_epic_materialized_views.sql b/analytics/src/analytics/integrations/etldb/migrations/versions/0015_add_deliverable_epic_materialized_views.sql new file mode 100644 index 0000000000..f186a43e13 --- /dev/null +++ b/analytics/src/analytics/integrations/etldb/migrations/versions/0015_add_deliverable_epic_materialized_views.sql @@ -0,0 +1,96 @@ +-- Precomputed per-quad deliverable ranking, current-state and independent +-- of any reporting period. Lets deliverable-ranking queries read a ranked +-- deliverable list directly instead of recomputing the window +-- function/anti-join over the full history tables on every query. + +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_deliverables_per_quad AS +WITH quad AS ( + SELECT id AS quad_id, name AS quad_name, ghid AS quad_ghid, start_date, end_date + FROM gh_quad +), +quad_deliverable_map AS ( + SELECT * + FROM ( + SELECT quad_id, deliverable_id, d_effective, + ROW_NUMBER() OVER (PARTITION BY deliverable_id ORDER BY d_effective DESC) AS ranked_order + FROM gh_deliverable_quad_map + ) history + WHERE history.ranked_order = 1 +), +all_deliverables AS ( + SELECT d.id AS deliverable_id, d.title AS deliverable_title, d.ghid AS deliverable_ghid + FROM gh_deliverable d + WHERE NOT EXISTS (SELECT 1 FROM gh_epic e WHERE e.ghid = d.ghid AND e.t_modified > d.t_modified) + AND NOT EXISTS (SELECT 1 FROM gh_issue i WHERE i.ghid = d.ghid AND i.t_modified > d.t_modified) +) +SELECT q.quad_id, q.quad_name, q.quad_ghid, + d.deliverable_id, d.deliverable_title, d.deliverable_ghid, + qdm.d_effective, + ROW_NUMBER() OVER (PARTITION BY q.quad_id ORDER BY d.deliverable_title) AS ranked_order +FROM quad q +JOIN quad_deliverable_map qdm ON q.quad_id = qdm.quad_id +JOIN all_deliverables d ON qdm.deliverable_id = d.deliverable_id; + +-- Each deliverable is mapped to exactly one (its latest) quad, so +-- deliverable_id alone is unique -- required for REFRESH ... CONCURRENTLY. +CREATE UNIQUE INDEX IF NOT EXISTS mv_dpq_i1 ON mv_deliverables_per_quad (deliverable_id); + +-- Precomputed latest epic-to-deliverable mapping, one row per epic holding +-- its most recent mapping by d_effective. Lets burndowns queries join +-- directly instead of recomputing a DISTINCT ON over the full mapping +-- history on every query. +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_latest_epic_deliverable_map AS +SELECT DISTINCT ON (edm.epic_id) edm.epic_id, e.ghid AS epic_ghid, edm.deliverable_id, edm.d_effective +FROM gh_epic_deliverable_map edm +JOIN gh_epic e ON edm.epic_id = e.id +ORDER BY edm.epic_id, edm.d_effective DESC; + +CREATE UNIQUE INDEX IF NOT EXISTS mv_ledm_i1 ON mv_latest_epic_deliverable_map (epic_id); + +-- Precomputed deliverable-to-issue membership (epic tree walk plus direct +-- children) joined against gh_issue_history and pre-aggregated by +-- (deliverable_id, day). Lets burndowns queries employ a single indexed +-- range scan instead of a recursive tree walk and full history join. + +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_deliverable_daily_burndown AS +WITH RECURSIVE +epics_in_deliverable AS ( + SELECT lem.epic_id, lem.epic_ghid, lem.deliverable_id + FROM mv_latest_epic_deliverable_map lem +), +epic_issue_tree AS ( + SELECT e.deliverable_id, i.id AS issue_id, i.ghid AS issue_ghid + FROM epics_in_deliverable e + JOIN gh_issue i ON i.parent_issue_ghid = e.epic_ghid + UNION ALL + SELECT eit.deliverable_id, i.id AS issue_id, i.ghid AS issue_ghid + FROM gh_issue i + JOIN epic_issue_tree eit ON i.parent_issue_ghid = eit.issue_ghid +), +direct_issues AS ( + SELECT d.id AS deliverable_id, i.id AS issue_id + FROM gh_deliverable d + JOIN gh_issue i ON i.parent_issue_ghid = d.ghid + WHERE NOT EXISTS ( + SELECT 1 + FROM mv_latest_epic_deliverable_map lem + JOIN gh_epic ep ON lem.epic_id = ep.id + WHERE ep.ghid = i.ghid + ) +), +deliverable_issue_map AS ( + SELECT deliverable_id, issue_id FROM epic_issue_tree + UNION ALL + SELECT deliverable_id, issue_id FROM direct_issues +) +SELECT dim.deliverable_id, + h.d_effective AS issue_day, + COUNT(DISTINCT h.issue_id) AS total_issues_opened, + COUNT(DISTINCT h.issue_id) FILTER (WHERE h.is_closed::BOOLEAN) AS total_issues_closed, + COALESCE(SUM(h.points), 0) AS total_points_opened, + COALESCE(SUM(h.points) FILTER (WHERE h.is_closed::BOOLEAN), 0) AS total_points_closed +FROM deliverable_issue_map dim +JOIN gh_issue_history h ON h.issue_id = dim.issue_id +GROUP BY dim.deliverable_id, h.d_effective; + +CREATE UNIQUE INDEX IF NOT EXISTS mv_ddb_i1 ON mv_deliverable_daily_burndown (deliverable_id, issue_day); diff --git a/analytics/src/analytics/integrations/metabase/sql/67-Sprint_Metrics/155-Sprint_Burnup_by_points.sql b/analytics/src/analytics/integrations/metabase/sql/67-Sprint_Metrics/155-Sprint_Burnup_by_points.sql index 151bf1fe02..5af2757844 100644 --- a/analytics/src/analytics/integrations/metabase/sql/67-Sprint_Metrics/155-Sprint_Burnup_by_points.sql +++ b/analytics/src/analytics/integrations/metabase/sql/67-Sprint_Metrics/155-Sprint_Burnup_by_points.sql @@ -10,20 +10,20 @@ WITH -- get project_id FROM project_data, gh_sprint WHERE gh_sprint.project_id = project_data.project_id - AND {{sprint_name}}), -- calculate points opened in the sprint - opened AS + AND {{sprint_name}}), -- calculate total sprint scope (points tracked) by day + SCOPE AS (SELECT gh_issue_history.d_effective AS DAY, - sum(gh_issue_history.points) AS total_opened + sum(gh_issue_history.points) AS total_scope FROM gh_issue_history, sprint_data WHERE gh_issue_history.sprint_id = sprint_data.sprint_id AND (gh_issue_history.d_effective >= sprint_data.sprint_start_date AND gh_issue_history.d_effective <= sprint_data.sprint_end_date) GROUP BY DAY - ORDER BY DAY), -- calculate points closed in the sprint - closed AS + ORDER BY DAY), -- calculate points completed in the sprint by day + completed AS (SELECT gh_issue_history.d_effective AS DAY, - sum(gh_issue_history.points) AS total_closed + sum(gh_issue_history.points) AS total_completed FROM gh_issue_history, sprint_data WHERE gh_issue_history.sprint_id = sprint_data.sprint_id @@ -31,15 +31,14 @@ WITH -- get project_id AND (gh_issue_history.d_effective >= sprint_data.sprint_start_date AND gh_issue_history.d_effective <= sprint_data.sprint_end_date) GROUP BY DAY - ORDER BY DAY), -- aggregate points opened and closed by day + ORDER BY DAY), -- aggregate scope and completed work by day (both climb toward total scope) totals AS - (SELECT opened.day AS DAY, - opened.total_opened, - closed.total_closed, - opened.total_opened - closed.total_closed AS total_remaining - FROM opened, - closed - WHERE opened.day = closed.day + (SELECT scope.day AS DAY, + scope.total_scope, + completed.total_completed + FROM SCOPE, + completed + WHERE scope.day = completed.day ORDER BY DAY) SELECT * -FROM totals \ No newline at end of file +FROM totals diff --git a/analytics/src/analytics/integrations/metabase/sql/67-Sprint_Metrics/156-Sprint_Burnup_by_issues.sql b/analytics/src/analytics/integrations/metabase/sql/67-Sprint_Metrics/156-Sprint_Burnup_by_issues.sql index 9d0ee7ff32..c5cdf504dc 100644 --- a/analytics/src/analytics/integrations/metabase/sql/67-Sprint_Metrics/156-Sprint_Burnup_by_issues.sql +++ b/analytics/src/analytics/integrations/metabase/sql/67-Sprint_Metrics/156-Sprint_Burnup_by_issues.sql @@ -10,20 +10,20 @@ WITH -- get project_id FROM project_data, gh_sprint WHERE gh_sprint.project_id = project_data.project_id - AND {{sprint_name}}), -- calculate issues opened in the sprint identified by sprint_id - opened AS + AND {{sprint_name}}), -- calculate total sprint scope (issues tracked) by day + SCOPE AS (SELECT gh_issue_history.d_effective AS DAY, - count(*) AS total_opened + count(*) AS total_scope FROM gh_issue_history, sprint_data WHERE gh_issue_history.sprint_id = sprint_data.sprint_id AND (gh_issue_history.d_effective >= sprint_data.sprint_start_date AND gh_issue_history.d_effective <= sprint_data.sprint_end_date) GROUP BY DAY - ORDER BY DAY), -- calculate issues closed in the sprint - closed AS + ORDER BY DAY), -- calculate issues completed in the sprint by day + completed AS (SELECT gh_issue_history.d_effective AS DAY, - count(*) AS total_closed + count(*) AS total_completed FROM gh_issue_history, sprint_data WHERE gh_issue_history.sprint_id = sprint_data.sprint_id @@ -31,15 +31,14 @@ WITH -- get project_id AND (gh_issue_history.d_effective >= sprint_data.sprint_start_date AND gh_issue_history.d_effective <= sprint_data.sprint_end_date) GROUP BY DAY - ORDER BY DAY), -- aggregate issues opened and closed by day + ORDER BY DAY), -- aggregate scope and completed work by day (both climb toward total scope) totals AS - (SELECT opened.day AS DAY, - opened.total_opened, - closed.total_closed, - opened.total_opened - closed.total_closed AS total_remaining - FROM opened, - closed - WHERE opened.day = closed.day + (SELECT scope.day AS DAY, + scope.total_scope, + completed.total_completed + FROM SCOPE, + completed + WHERE scope.day = completed.day ORDER BY DAY) SELECT * -FROM totals \ No newline at end of file +FROM totals diff --git a/analytics/src/analytics/integrations/metabase/sql/69-Delivery_Metrics/212-Deliverable_Burnup_Issue_Count.sql b/analytics/src/analytics/integrations/metabase/sql/69-Delivery_Metrics/212-Deliverable_Burnup_Issue_Count.sql index 14554e6b89..5c5ce27af5 100644 --- a/analytics/src/analytics/integrations/metabase/sql/69-Delivery_Metrics/212-Deliverable_Burnup_Issue_Count.sql +++ b/analytics/src/analytics/integrations/metabase/sql/69-Delivery_Metrics/212-Deliverable_Burnup_Issue_Count.sql @@ -65,33 +65,32 @@ WITH RECURSIVE -- 1. Get reporting period dynamically (SELECT * FROM epic_issue_tree UNION ALL SELECT * - FROM direct_issues), -- 9. Calculate issues opened in the given time period - opened AS + FROM direct_issues), -- 9. Calculate total issue scope (issues tracked) in the given time period + scope AS (SELECT h.d_effective AS issue_day, - COUNT(DISTINCT h.issue_id) AS total_opened + COUNT(DISTINCT h.issue_id) AS total_scope FROM gh_issue_history h JOIN combined_issues ci ON h.issue_id = ci.issue_id CROSS JOIN reporting_period WHERE h.d_effective BETWEEN reporting_period.start_date AND reporting_period.end_date GROUP BY h.d_effective - ORDER BY h.d_effective), -- 10. Calculate issues closed in the given time period - closed AS + ORDER BY h.d_effective), -- 10. Calculate issues completed in the given time period + completed AS (SELECT h.d_effective AS issue_day, - COUNT(DISTINCT h.issue_id) AS total_closed + COUNT(DISTINCT h.issue_id) AS total_completed FROM gh_issue_history h JOIN combined_issues ci ON h.issue_id = ci.issue_id CROSS JOIN reporting_period WHERE h.is_closed::BOOLEAN = TRUE AND h.d_effective BETWEEN reporting_period.start_date AND reporting_period.end_date GROUP BY h.d_effective - ORDER BY h.d_effective), -- 11. Aggregate issues opened and closed by day + ORDER BY h.d_effective), -- 11. Aggregate scope and completed work by day (both climb toward total scope) totals AS - (SELECT COALESCE(o.issue_day, c.issue_day) AS issue_day, - COALESCE(o.total_opened, 0) AS total_opened, - COALESCE(c.total_closed, 0) AS total_closed, - COALESCE(o.total_opened, 0) - COALESCE(c.total_closed, 0) AS total_remaining - FROM opened o - FULL OUTER JOIN closed c ON o.issue_day = c.issue_day + (SELECT COALESCE(s.issue_day, c.issue_day) AS issue_day, + COALESCE(s.total_scope, 0) AS total_scope, + COALESCE(c.total_completed, 0) AS total_completed + FROM scope s + FULL OUTER JOIN completed c ON s.issue_day = c.issue_day ORDER BY issue_day) SELECT * FROM totals; \ No newline at end of file diff --git a/analytics/src/analytics/integrations/metabase/sql/69-Delivery_Metrics/213-Deliverable_Burnup_Points.sql b/analytics/src/analytics/integrations/metabase/sql/69-Delivery_Metrics/213-Deliverable_Burnup_Points.sql index f722689163..cec88bbb68 100644 --- a/analytics/src/analytics/integrations/metabase/sql/69-Delivery_Metrics/213-Deliverable_Burnup_Points.sql +++ b/analytics/src/analytics/integrations/metabase/sql/69-Delivery_Metrics/213-Deliverable_Burnup_Points.sql @@ -65,33 +65,32 @@ WITH RECURSIVE -- 1. Get reporting period dynamically (SELECT * FROM epic_issue_tree UNION ALL SELECT * - FROM direct_issues), -- 9. Calculate points opened in the given time period - opened AS + FROM direct_issues), -- 9. Calculate total points scope (points tracked) in the given time period + scope AS (SELECT h.d_effective AS issue_day, - SUM(h.points) AS total_opened + SUM(h.points) AS total_scope FROM gh_issue_history h JOIN combined_issues ci ON h.issue_id = ci.issue_id CROSS JOIN reporting_period WHERE h.d_effective BETWEEN reporting_period.start_date AND reporting_period.end_date GROUP BY h.d_effective - ORDER BY h.d_effective), -- 10. Calculate points closed in the given time period - closed AS + ORDER BY h.d_effective), -- 10. Calculate points completed in the given time period + completed AS (SELECT h.d_effective AS issue_day, - SUM(h.points) AS total_closed + SUM(h.points) AS total_completed FROM gh_issue_history h JOIN combined_issues ci ON h.issue_id = ci.issue_id CROSS JOIN reporting_period WHERE h.is_closed::BOOLEAN = TRUE AND h.d_effective BETWEEN reporting_period.start_date AND reporting_period.end_date GROUP BY h.d_effective - ORDER BY h.d_effective), -- 11. Aggregate points opened and closed by day + ORDER BY h.d_effective), -- 11. Aggregate scope and completed work by day (both climb toward total scope) totals AS - (SELECT COALESCE(o.issue_day, c.issue_day) AS issue_day, - COALESCE(o.total_opened, 0) AS total_opened, - COALESCE(c.total_closed, 0) AS total_closed, - COALESCE(o.total_opened, 0) - COALESCE(c.total_closed, 0) AS total_remaining - FROM opened o - FULL OUTER JOIN closed c ON o.issue_day = c.issue_day + (SELECT COALESCE(s.issue_day, c.issue_day) AS issue_day, + COALESCE(s.total_scope, 0) AS total_scope, + COALESCE(c.total_completed, 0) AS total_completed + FROM scope s + FULL OUTER JOIN completed c ON s.issue_day = c.issue_day ORDER BY issue_day) SELECT * FROM totals; \ No newline at end of file