From 46b26c2add1c8c672976404389ce94d0218fa2ab Mon Sep 17 00:00:00 2001 From: Travis Long Date: Wed, 22 Jul 2026 10:04:49 -0400 Subject: [PATCH 1/2] Bug 2006133 - Add templated queries for data anomaly investigation --- src/data/gleanSql.js | 324 ++++++++++++++++++++++++++++++++++ src/pages/MetricDetail.svelte | 47 +++++ tests/data.gleanSql.test.js | 254 ++++++++++++++++++++++++++ 3 files changed, 625 insertions(+) create mode 100644 tests/data.gleanSql.test.js diff --git a/src/data/gleanSql.js b/src/data/gleanSql.js index f9b24c4be..34326eec9 100644 --- a/src/data/gleanSql.js +++ b/src/data/gleanSql.js @@ -189,3 +189,327 @@ LIMIT 10`; export const getGleanPingQuerySTMOTemplateUrl = (table) => `https://sql.telemetry.mozilla.org/queries/110682/source?p_table=${table}`; + +// The Glean Dictionary knows each metric's type, so we can pick a sensible way +// to surface its value in the investigation query instead of leaving the raw +// array/struct/JSON column for the user to figure out. Given a metric's type +// and its BigQuery column, return: +// select: the SELECT statement for the value. +// crossJoin: any CROSS JOIN UNNEST needed to examine it or an empty string. +// The BigQuery shapes below match how etl/looker.py builds its metric explores. +const HEADER = " -- Metric under investigation:"; + +// Object metrics are stored as a single JSON column. When the Dictionary knows +// the metric's `structure`, exposed by the probe-info service, we can offer the +// fields as ready-made JSON_VALUE paths. Return { paths, skippedMap, skippedArray }: +// paths: [{ path: "$.a.b", alias: "a_b" }] for scalars reachable by a +// plain object path. +// skippedMap: a Glean {key, value} object was skipped. +// skippedArray: an array was skipped because it would need UNNEST/JSON_QUERY_ARRAY. +const SCALAR_JSON_TYPES = new Set(["string", "number", "boolean", "oneOf"]); + +const isObjectMap = (node) => { + const props = node.properties ? Object.keys(node.properties) : []; + return props.length === 2 && props.includes("key") && props.includes("value"); +}; + +const collectObjectScalarPaths = (structure) => { + const paths = []; + const flags = { skippedMap: false, skippedArray: false }; + const walk = (node, path, aliasParts) => { + if (!node || typeof node !== "object") return; + if (SCALAR_JSON_TYPES.has(node.type)) { + paths.push({ path, alias: aliasParts.join("_") }); + } else if (node.type === "array") { + flags.skippedArray = true; + } else if (node.type === "object" && node.properties) { + if (isObjectMap(node)) { + flags.skippedMap = true; + } else { + Object.entries(node.properties).forEach(([name, child]) => + walk(child, `${path}.${name}`, [...aliasParts, name]) + ); + } + } + }; + if (structure && structure.type === "array") { + flags.skippedArray = true; + } else if (structure && structure.type === "object" && structure.properties) { + Object.entries(structure.properties).forEach(([name, child]) => + walk(child, `$.${name}`, [name]) + ); + } + return { paths, ...flags }; +}; + +const getMetricValueSql = (metricType, columnName, structure) => { + // Alias off the last path segment, e.g. metrics.counter.foo_bar -> foo_bar. + const alias = columnName.split(".").pop(); + const unnest = (label) => `\nCROSS JOIN UNNEST(${columnName}) AS ${label}`; + + switch (metricType) { + // SUM and AVG numeric scalar metric types + case "counter": + case "quantity": + return { + select: `${HEADER} totalled across the pings in each group. + SUM(${columnName}) AS ${alias}_sum, + AVG(${columnName}) AS ${alias}_avg,`, + crossJoin: "", + }; + case "timespan": + // STRUCT. + return { + select: `${HEADER} timespan total per group (units are in ${columnName}.time_unit). + SUM(${columnName}.value) AS ${alias}_sum,`, + crossJoin: "", + }; + case "rate": + // STRUCT. + return { + select: `${HEADER} rate numerator/denominator totalled per group. + SUM(${columnName}.numerator) AS ${alias}_numerator, + SUM(${columnName}.denominator) AS ${alias}_denominator,`, + crossJoin: "", + }; + case "timing_distribution": + case "memory_distribution": + case "custom_distribution": + // STRUCT>. Sum is a good starting point; + // percentiles need the .values histogram. + return { + select: `${HEADER} distribution sum per group (a starting point; pull + -- percentiles from ${columnName}.values for more detail). + SUM(${columnName}.sum) AS ${alias}_sum,`, + crossJoin: "", + }; + + // For low-cardinality metric types, group by the value directly + case "boolean": + case "string": + case "datetime": + return { + select: `${HEADER} grouped by its value. + ${columnName} AS ${alias},`, + crossJoin: "", + }; + + // For metric types likely to have high-cardinality, count distinct rather than group by + case "uuid": + case "url": + case "text": + return { + select: `${HEADER} high-cardinality ${metricType}; count distinct values + -- (swap for "${columnName} AS ${alias}," to list them instead). + COUNT(DISTINCT ${columnName}) AS distinct_${alias},`, + crossJoin: "", + }; + + // Labeled metrics we UNNEST and break down by label + case "labeled_counter": + case "labeled_quantity": + return { + select: `${HEADER} broken down by label (counts are per ping-label row). + ${alias}_label.key AS ${alias}_label, + SUM(${alias}_label.value) AS ${alias}_sum,`, + crossJoin: unnest(`${alias}_label`), + }; + case "labeled_string": + case "labeled_boolean": + return { + select: `${HEADER} broken down by label (counts are per ping-label row). + ${alias}_label.key AS ${alias}_label, + ${alias}_label.value AS ${alias}_value,`, + crossJoin: unnest(`${alias}_label`), + }; + case "labeled_custom_distribution": + case "labeled_timing_distribution": + case "labeled_memory_distribution": + return { + select: `${HEADER} distribution sum per label (counts are per ping-label row). + ${alias}_label.key AS ${alias}_label, + SUM(${alias}_label.value.sum) AS ${alias}_sum,`, + crossJoin: unnest(`${alias}_label`), + }; + case "dual_labeled_counter": + // ARRAY>>>. + return { + select: `${HEADER} broken down by both label keys (counts are per row). + ${alias}_key.key AS ${alias}_key, + ${alias}_category.key AS ${alias}_category, + SUM(${alias}_category.value) AS ${alias}_sum,`, + crossJoin: `${unnest( + `${alias}_key` + )}\nCROSS JOIN UNNEST(${alias}_key.value) AS ${alias}_category`, + }; + case "string_list": + // ARRAY. + return { + select: `${HEADER} broken down by list item (counts are per ping-item row). + ${alias}_item,`, + crossJoin: unnest(`${alias}_item`), + }; + + // Events live in events_stream and thus are handled per-row + case "event": + return { + select: `${HEADER} grouped by the event name. + event, + -- event_extra is JSON: extract a field with JSON_VALUE(event_extra.the_key) + -- and add it here if you need to segment by it. + -- event_extra,`, + crossJoin: "", + }; + + // Object is a JSON column type, so we need to look at the structure + // to identify scalar fields we can extract with JSON_VALUE. If the structure is + // missing or has no scalar fields, fall back to a generic hint. + case "object": { + const { paths, skippedMap, skippedArray } = + collectObjectScalarPaths(structure); + if (paths.length) { + const options = paths + .map( + ({ path, alias: leaf }) => + ` -- JSON_VALUE(${columnName}, '${path}') AS ${leaf},` + ) + .join("\n"); + const skipped = [ + skippedMap && "map (key/value) fields", + skippedArray && "array fields", + ].filter(Boolean); + const skipNote = skipped.length + ? `\n -- (${skipped.join( + " and " + )} need UNNEST/JSON_QUERY_ARRAY and are omitted here)` + : ""; + return { + select: `${HEADER} object metric is JSON. Uncomment a field below to + -- segment by it (scalar fields taken from the metric's structure):${skipNote} +${options}`, + crossJoin: "", + }; + } + // This field is a top-level array, a map-only object, or there's no + // structure metadata available, so we fall back to a generic hint. + const hint = skippedArray + ? `object metric is a JSON array; UNNEST JSON_QUERY_ARRAY(${columnName}) + -- and pull scalars from each element with JSON_VALUE(item, '$.field').` + : `object metric is JSON; extract a scalar to segment by + -- with JSON_VALUE(${columnName}, '$.field') and add it here.`; + return { + select: `${HEADER} ${hint} + -- ${columnName},`, + crossJoin: "", + }; + } + default: + return { + select: `${HEADER} this ${metricType} column can't be summarised + -- automatically; adapt the reference below as needed. + -- ${columnName},`, + crossJoin: "", + }; + } +}; + +// Builds a single "starter" query that helps investigate data anomalies by +// slicing a metric's pings across the diagnostic dimensions described in +// https://mozilla.github.io/glean/book/user/howto/investigating-data-issues/investigating-data-issues.html +// The metric value is selected up front (it's what we're investigating). Each +// diagnostic dimension is included as a commented-out block that the +// investigator uncomments to segment by. GROUP BY ALL then groups by whatever +// is selected so there's no separate GROUP BY list to maintain. +export const getGleanInvestigationQuery = ( + metricType, + table, + columnName, + eventInfo, + structure +) => { + const isEvent = metricType === "event"; + // Event metrics live in the `events_stream` table rather than the ping table, + // matching the behavior of the other event query generators above. + let fromTable = table; + if (isEvent) { + // Change `some_dataset.some_table` to `some_dataset.events_stream`, + // matching getSQLResource() in MetricDetail.svelte. + const [dataset] = table.split("."); + fromTable = `${dataset}.events_stream`; + } + + const eventFilter = isEvent + ? `\n AND event = '${eventInfo.category}.${eventInfo.name}'` + : `\n -- Restrict to pings that actually carry this metric.\n -- AND ${columnName} IS NOT NULL`; + + // The metric value is the main thing we're slicing, so surface it by default + // using a strategy chosen from its type (see getMetricValueSql above). + const { select: metricSelect, crossJoin: metricCrossJoin } = + getMetricValueSql(metricType, columnName, structure); + + return ` +-- Auto-generated by the Glean Dictionary. +-- Investigate a data anomaly by slicing this metric's pings across the +-- diagnostic dimensions from the Glean data-investigation guide: +-- https://mozilla.github.io/glean/book/user/howto/investigating-data-issues/investigating-data-issues.html +-- +-- To segment by a dimension, uncomment it in the SELECT block below (GROUP BY +-- ALL picks it up automatically), then look for the segment(s) that explain +-- the anomaly. + +SELECT + DATE(submission_timestamp) AS submission_date, + COUNT(*) AS ping_count, + COUNT(DISTINCT client_info.client_id) AS client_count, +${metricSelect} + -- 1. Countries: geographical patterns (national holidays, bot-prone regions). + -- metadata.geo.country, + -- 2. ISP: finer-grained than country; a single ISP can indicate automation. + -- Tip: uncomment the HAVING clause below to drop small ISPs. + -- metadata.isp.name, + -- 3. Product version / build: did the anomaly start with a release? An + -- unknown build could be a clone, fork, or side-load. + -- client_info.app_display_version, + -- client_info.app_build, + -- 4. Glean SDK version: did the anomaly start after a Glean update? + -- client_info.telemetry_sdk_build, + -- 6. OS / platform version (Android only: client_info.android_sdk_version). + -- client_info.os_version, + -- client_info.android_sdk_version, + -- 9. Hardware (mobile only): is the issue specific to certain devices? + -- client_info.device_manufacturer, + -- client_info.device_model, + -- 10. Architecture: is the issue specific to a build configuration? + -- client_info.architecture, + -- 11. Ping reason: is the anomaly tied to a specific submission reason? + -- ping_info.reason, + -- 7. Time gaps: is the delay from collection to submission reasonable? + -- TIMESTAMP_DIFF(submission_timestamp, ping_info.parsed_end_time, HOUR) AS submission_delay_hours, + -- ping_info.parsed_start_time, + -- ping_info.parsed_end_time, +FROM + ${fromTable} AS m${metricCrossJoin} +WHERE + -- Look at the last two weeks so the anomaly stands out against a baseline. + -- https://docs.telemetry.mozilla.org/cookbooks/bigquery/querying.html#table-layout-and-naming + DATE(submission_timestamp) >= DATE_SUB(CURRENT_DATE(), INTERVAL 14 DAY)${eventFilter} +-- GROUP BY ALL groups by every non-aggregated column selected above, so +-- uncommenting a dimension is all that's needed. Array/JSON/struct columns +-- (labeled metrics, distributions, string lists, event_extra, ...) can't be +-- grouped directly: UNNEST or extract a scalar from them first. +GROUP BY ALL +-- HAVING filters the grouped rows (it goes after GROUP BY, unlike WHERE). +-- Uncomment to drop small/noisy segments, e.g. ISPs with very few pings: +-- HAVING ping_count > 5000 +ORDER BY + submission_date DESC +-- IMPORTANT: Remove the limit clause when the query is ready. +LIMIT 100 + +-- Dimensions that aren't simple slices (see the guide for details): +-- * 5. Other library version changes (Application Services, Gecko, Viaduct, rkv). +-- * 8. Glean errors: check the *_error_* metrics and network/ingestion errors. +-- https://mozilla.github.io/glean/book/user/metrics/error-reporting.html +-- * 12. No data: verify send_in_pings, metric lifetime, active recording code +-- path, and any Server Knobs experiments/rollouts.`; +}; diff --git a/src/pages/MetricDetail.svelte b/src/pages/MetricDetail.svelte index b42a05ebc..72a661de5 100644 --- a/src/pages/MetricDetail.svelte +++ b/src/pages/MetricDetail.svelte @@ -54,6 +54,7 @@ getGleanLabeledCounterQuerySTMOTemplateUrl, getGleanDualLabeledCounterQuerySTMOTemplateUrl, getGleanQuerySTMOTemplateUrl, + getGleanInvestigationQuery, } from "../data/gleanSql"; export let params; @@ -688,6 +689,52 @@ + + + + + + + + + +
+ Diagnostic query + + +
+ Investigating an anomaly? Generate a query that slices this metric's + data across the + + recommended diagnostic dimensions + + ➡   +
+
+ +
+
{/if} {:catch} diff --git a/tests/data.gleanSql.test.js b/tests/data.gleanSql.test.js new file mode 100644 index 000000000..92c53457e --- /dev/null +++ b/tests/data.gleanSql.test.js @@ -0,0 +1,254 @@ +import { getGleanInvestigationQuery } from "../src/data/gleanSql"; + +describe("getGleanInvestigationQuery", () => { + it("includes the diagnostic dimensions and doc link for a scalar metric", () => { + const query = getGleanInvestigationQuery( + "counter", + "fenix.metrics", + "metrics.counter.some_metric", + undefined + ); + + // Diagnostic dimensions from the investigation guide. + expect(query).toContain("metadata.geo.country"); + expect(query).toContain("metadata.isp.name"); + expect(query).toContain("client_info.app_display_version"); + expect(query).toContain("client_info.telemetry_sdk_build"); + expect(query).toContain("client_info.os_version"); + expect(query).toContain("client_info.architecture"); + expect(query).toContain("ping_info.reason"); + + // Queries the ping table and groups with GROUP BY ALL. + expect(query).toContain("fenix.metrics AS m"); + expect(query).toContain("GROUP BY ALL"); + + // Reports both ping and distinct-client volume. + expect(query).toContain("COUNT(*) AS ping_count"); + expect(query).toContain( + "COUNT(DISTINCT client_info.client_id) AS client_count" + ); + + // Shows where a HAVING clause goes (commented out). + expect(query).toContain("-- HAVING ping_count > 5000"); + + // Link back to the investigation guide. + expect(query).toContain( + "investigating-data-issues/investigating-data-issues.html" + ); + + // No active event filter for a non-event metric. + expect(query).not.toMatch(/^\s*AND event = /m); + }); + + it("totals numeric scalar metrics", () => { + const query = getGleanInvestigationQuery( + "counter", + "fenix.metrics", + "metrics.counter.some_metric", + undefined + ); + expect(query).toContain( + "SUM(metrics.counter.some_metric) AS some_metric_sum" + ); + expect(query).toContain( + "AVG(metrics.counter.some_metric) AS some_metric_avg" + ); + }); + + it("groups low-cardinality scalars by value", () => { + const query = getGleanInvestigationQuery( + "boolean", + "fenix.metrics", + "metrics.boolean.some_flag", + undefined + ); + expect(query).toMatch(/^ {2}metrics\.boolean\.some_flag AS some_flag,$/m); + }); + + it("counts distinct values for high-cardinality scalars", () => { + const query = getGleanInvestigationQuery( + "uuid", + "fenix.metrics", + "metrics.uuid.some_id", + undefined + ); + expect(query).toContain( + "COUNT(DISTINCT metrics.uuid.some_id) AS distinct_some_id" + ); + }); + + it("extracts the sum from distribution metrics", () => { + const query = getGleanInvestigationQuery( + "timing_distribution", + "fenix.metrics", + "metrics.timing_distribution.load_time", + undefined + ); + expect(query).toContain( + "SUM(metrics.timing_distribution.load_time.sum) AS load_time_sum" + ); + }); + + it("UNNESTs labeled metrics and breaks them down by label", () => { + const query = getGleanInvestigationQuery( + "labeled_counter", + "fenix.metrics", + "metrics.labeled_counter.some_metric", + undefined + ); + expect(query).toContain( + "CROSS JOIN UNNEST(metrics.labeled_counter.some_metric) AS some_metric_label" + ); + expect(query).toContain("some_metric_label.key AS some_metric_label"); + expect(query).toContain("SUM(some_metric_label.value) AS some_metric_sum"); + }); + + it("double-UNNESTs dual labeled counters", () => { + const query = getGleanInvestigationQuery( + "dual_labeled_counter", + "fenix.metrics", + "metrics.dual_labeled_counter.some_metric", + undefined + ); + expect(query).toContain( + "CROSS JOIN UNNEST(metrics.dual_labeled_counter.some_metric) AS some_metric_key" + ); + expect(query).toContain( + "CROSS JOIN UNNEST(some_metric_key.value) AS some_metric_category" + ); + }); + + it("offers JSON_VALUE paths for an object metric's scalar fields, skipping maps", () => { + // Shape of glean.internal.metrics.server_knobs_config. + const structure = { + type: "object", + properties: { + metrics_enabled: { + type: "object", + properties: { key: { type: "string" }, value: { type: "boolean" } }, + }, + pings_enabled: { + type: "object", + properties: { key: { type: "string" }, value: { type: "boolean" } }, + }, + event_threshold: { type: "number" }, + session_sample_rate: { type: "number" }, + }, + }; + const query = getGleanInvestigationQuery( + "object", + "fenix.metrics", + "metrics.object.server_knobs_config", + undefined, + structure + ); + + // Scalar leaves become concrete JSON_VALUE options. + expect(query).toContain( + "-- JSON_VALUE(metrics.object.server_knobs_config, '$.event_threshold') AS event_threshold," + ); + expect(query).toContain( + "-- JSON_VALUE(metrics.object.server_knobs_config, '$.session_sample_rate') AS session_sample_rate," + ); + // Map (key/value) fields are skipped and called out. + expect(query).not.toContain("metrics_enabled"); + expect(query).toContain("map (key/value) fields"); + }); + + it("recurses into nested objects and aliases by the full path", () => { + const structure = { + type: "object", + properties: { + settings: { + type: "object", + properties: { enabled: { type: "boolean" } }, + }, + }, + }; + const query = getGleanInvestigationQuery( + "object", + "fenix.metrics", + "metrics.object.config", + undefined, + structure + ); + expect(query).toContain( + "-- JSON_VALUE(metrics.object.config, '$.settings.enabled') AS settings_enabled," + ); + }); + + it("treats a oneOf field as a scalar JSON_VALUE path", () => { + // Glean object metrics allow `oneOf`, whose subtypes are restricted to + // scalars (string/number/boolean), so it is JSON_VALUE-extractable. + const structure = { + type: "object", + properties: { + state: { + type: "oneOf", + subtypes: [{ type: "string" }, { type: "number" }], + }, + }, + }; + const query = getGleanInvestigationQuery( + "object", + "fenix.metrics", + "metrics.object.config", + undefined, + structure + ); + expect(query).toContain( + "-- JSON_VALUE(metrics.object.config, '$.state') AS state," + ); + }); + + it("hints at UNNEST for object metrics whose structure is a JSON array", () => { + // Shape of glean.health.data_directory_info. + const structure = { + type: "array", + items: { + type: "object", + properties: { dir_name: { type: "string" } }, + }, + }; + const query = getGleanInvestigationQuery( + "object", + "fenix.metrics", + "metrics.object.data_directory_info", + undefined, + structure + ); + expect(query).toContain( + "JSON_QUERY_ARRAY(metrics.object.data_directory_info)" + ); + }); + + it("falls back to a generic JSON_VALUE hint when no structure is available", () => { + const query = getGleanInvestigationQuery( + "object", + "fenix.metrics", + "metrics.object.some_object", + undefined + ); + expect(query).not.toMatch(/^ {2}metrics\.object\.some_object,$/m); + expect(query).toContain( + "JSON_VALUE(metrics.object.some_object, '$.field')" + ); + }); + + it("targets events_stream and filters by event for an event metric", () => { + const query = getGleanInvestigationQuery( + "event", + "fenix.events", + "events", + { + category: "activation", + name: "identifier", + } + ); + + expect(query).toContain("fenix.events_stream AS m"); + expect(query).toContain("AND event = 'activation.identifier'"); + // Still slices across the same diagnostic dimensions. + expect(query).toContain("metadata.geo.country"); + }); +}); From e791734f57ea02064af2e8d616dc991311bca0d4 Mon Sep 17 00:00:00 2001 From: Travis Long Date: Thu, 23 Jul 2026 10:56:58 -0400 Subject: [PATCH 2/2] Update gleanSql.js Add a comment explaining that pings with no client_ids will have a zero for client_count --- src/data/gleanSql.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/data/gleanSql.js b/src/data/gleanSql.js index 34326eec9..679035845 100644 --- a/src/data/gleanSql.js +++ b/src/data/gleanSql.js @@ -460,6 +460,7 @@ export const getGleanInvestigationQuery = ( SELECT DATE(submission_timestamp) AS submission_date, COUNT(*) AS ping_count, + -- Note: pings that omit client_id in their configuration will return 0 for client_count. COUNT(DISTINCT client_info.client_id) AS client_count, ${metricSelect} -- 1. Countries: geographical patterns (national holidays, bot-prone regions).