From fd286f56768f002f40bdc42deec77cbf4e6faf62 Mon Sep 17 00:00:00 2001 From: William Callahan Date: Tue, 4 Aug 2026 10:02:33 -0700 Subject: [PATCH 1/2] fix(embeddings): keep the keep-alive failure escalation out of ERROR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A background keep-alive probe failure is a monitoring signal, not a user-facing request failure — the class doc already says so — yet the second consecutive failure logged ERROR. Every ERROR line feeds the critical java-chat-error-log Grafana alert, so a routine LLM gateway redeploy (minutes of 503s while the compose stack recreates) paged java-chat even though the app kept serving. Provider-health paging is owned by the dependencies health group (java-chat-dependencies-degraded) and the gateway's own availability rules, so the repeated-failure escalation now logs WARN like the first failure and the slow-probe loop. ERROR stays reserved for user-facing failures. Health still flips DOWN immediately; only the log level moves. --- .../javachat/service/EmbeddingModelKeepAlive.java | 12 ++++++++++-- .../service/EmbeddingModelKeepAliveTest.java | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/williamcallahan/javachat/service/EmbeddingModelKeepAlive.java b/src/main/java/com/williamcallahan/javachat/service/EmbeddingModelKeepAlive.java index 1897c15a..22e43093 100644 --- a/src/main/java/com/williamcallahan/javachat/service/EmbeddingModelKeepAlive.java +++ b/src/main/java/com/williamcallahan/javachat/service/EmbeddingModelKeepAlive.java @@ -27,7 +27,15 @@ public class EmbeddingModelKeepAlive implements HealthIndicator { /** Probe latency above this marks a slow provider response without inferring its remote cause. */ private static final long SLOW_PROBE_THRESHOLD_MILLIS = 5_000L; - /** Escalates only after a probe condition repeats on the next observation. */ + /** + * Escalates only after a probe condition repeats on the next observation. + * + *

The escalation stays at WARN even when repeated: a background probe failure is a + * monitoring signal, never a user-facing request failure, and ERROR lines feed the + * critical java-chat-error-log Grafana alert. Provider-health paging is owned by the + * dependencies health group (java-chat-dependencies-degraded) and the gateway's own + * availability rules, so a routine gateway redeploy must not page here.

+ */ private static final int REPEATED_PROBE_ALERT_COUNT = 2; private static final long NANOS_PER_MILLISECOND = 1_000_000L; @@ -178,7 +186,7 @@ private void recordFailure( .log(() -> "event=embedding_model_probe_failed outcome=failure model=" + logSafeModelName + " durationMs=" + probeDurationMillis + " consecutiveFailures=" + consecutiveFailureCount); } else if (consecutiveFailureCount == REPEATED_PROBE_ALERT_COUNT) { - log.atError() + log.atWarn() .setCause(embeddingUnavailableException) .log(() -> "event=embedding_model_probe_failure_loop outcome=failure model=" + logSafeModelName + " durationMs=" + probeDurationMillis + " consecutiveFailures=" + consecutiveFailureCount); diff --git a/src/test/java/com/williamcallahan/javachat/service/EmbeddingModelKeepAliveTest.java b/src/test/java/com/williamcallahan/javachat/service/EmbeddingModelKeepAliveTest.java index fad2c6c9..50aa52fe 100644 --- a/src/test/java/com/williamcallahan/javachat/service/EmbeddingModelKeepAliveTest.java +++ b/src/test/java/com/williamcallahan/javachat/service/EmbeddingModelKeepAliveTest.java @@ -124,7 +124,8 @@ void repeatedFailuresEscalateOnceAndRecoveryRestoresHealth() { assertEquals(Status.DOWN, keepAlive.health().getStatus()); assertEquals(1, eventCount(Level.WARN, "event=embedding_model_probe_failed")); - assertEquals(1, eventCount(Level.ERROR, "event=embedding_model_probe_failure_loop")); + assertEquals(1, eventCount(Level.WARN, "event=embedding_model_probe_failure_loop")); + assertEquals(0, eventCount(Level.ERROR, "event=embedding_model_probe_")); keepAlive.retryUnavailableEmbeddingModel(); From 599118c4744a0ff6bc3589a162c841817b6e9887 Mon Sep 17 00:00:00 2001 From: William Callahan Date: Tue, 4 Aug 2026 10:03:09 -0700 Subject: [PATCH 2/2] fix(dev): drop DEBUG Spring MVC plumbing from the deployed dev log stream dev.javachat.ai runs the dev profile, so logging.level.org.springframework.web=DEBUG shipped to Loki: a 5,000-line 3h sample held ~4,975 framework request-plumbing lines against 24 application lines (73,705 lines/24h vs production's 1,233), burying the EmbeddingModelKeepAlive events the java-chat Loki alert rules parse. Raise the level locally when debugging Spring MVC itself rather than for every deployed request. --- src/main/resources/application-dev.properties | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index 4a9bf300..d9e225b9 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -39,9 +39,17 @@ spring.thymeleaf.cache=false logging.level.org.springframework.boot.devtools=DEBUG # Development-specific overrides -# More verbose logging for development +# More verbose logging for development. +# +# `dev` is also the profile the deployed dev.javachat.ai runs under, so anything set here ships +# to Loki. org.springframework.web at DEBUG was 99.5% of this service's log volume — a 5,000-line +# sample over 3h on 2026-08-04 held 1,991 DispatcherServlet + 1,990 HttpEntityMethodProcessor + +# 994 RequestResponseBodyMethodProcessor lines against 24 application lines, essentially all of it +# request plumbing for the 15s /actuator/prometheus scrape. It buried the signal that matters +# (EmbeddingModelKeepAlive, which the java-chat Loki alert rules parse) and cost 73,705 lines/24h +# against production's 1,233. Raise it locally when debugging Spring MVC itself, e.g. +# `--logging.level.org.springframework.web=DEBUG`, rather than for every deployed request. logging.level.com.williamcallahan.javachat=DEBUG -logging.level.org.springframework.web=DEBUG logging.level.org.springframework.ai=DEBUG # CORS for Vite dev server (port 5173) + local Spring Boot (port 8085) + deployed dev host.