From aac4f2b76db8a62ecccdf72cb559672bd079662d Mon Sep 17 00:00:00 2001 From: Ryan Min Date: Tue, 1 Sep 2026 10:28:15 +0900 Subject: [PATCH 01/13] test: materialize basic strategy occurrences --- .../BasicExecutionPlanCompletionTest.java | 37 ++++++++++++++++++- .../BasicStrategyWarningAnalyzerTest.java | 13 +++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicExecutionPlanCompletionTest.java b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicExecutionPlanCompletionTest.java index 45160409..26ee4ca9 100644 --- a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicExecutionPlanCompletionTest.java +++ b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicExecutionPlanCompletionTest.java @@ -19,6 +19,40 @@ class BasicExecutionPlanCompletionTest { private static final UUID AAPL = UUID.fromString("10000000-0000-4000-8000-000000000001"); private static final UUID MSFT = UUID.fromString("20000000-0000-4000-8000-000000000001"); + @Test + void preservesOccurrenceArgumentsSideResolutionAndInstrumentSelection() throws Exception { + String semantic = """ + {"catalogId":"0f5a0000-0000-4000-8000-000000000001","groups":[ + {"id":"sell:bounds","allocationGroupId":"sell","container":"SELL","evaluationMode":"INDEPENDENT","allocationMode":"EQUAL", + "instrumentIds":["20000000-0000-4000-8000-000000000001","10000000-0000-4000-8000-000000000001"], + "blocks":[ + {"id":"lower","elementCode":"TEST_CONDITION","parameters":{"resolution":"1h","operator":"GTE","thresholdPercent":"10"}}, + {"id":"upper","elementCode":"TEST_CONDITION","parameters":{"resolution":"1h","operator":"LT","thresholdPercent":"5"}}, + {"id":"order","elementCode":"BASIC_EQUAL_ALLOCATION_ORDER","parameters":{"orderPercent":"25","maxPositionPercent":"40","executionMode":"1회만","waitMode":"조건 재충족","waitInterval":"1","maxExecutions":"1"}}], + "connections":[ + {"fromBlockId":"lower","outputPort":"passed","toBlockId":"upper","inputPort":"passed"}, + {"fromBlockId":"upper","outputPort":"passed","toBlockId":"order","inputPort":"passed"}]} + ]} + """; + String canonical = StrategyDocumentJson.canonicalize(semantic); + var document = new StrategyDocument( + UUID.randomUUID(), canonical, "{}", "basic-semantic/v1", "basic-presentation/v1", + StrategyDocumentJson.sha256(canonical), StrategyDocumentJson.sha256("{}"), 1, + Instant.parse("2026-08-25T00:00:00Z"), Instant.parse("2026-08-25T00:00:00Z")); + + JsonNode flow = JSON.readTree(new BasicExecutionPlanCompiler().compile( + UUID.randomUUID(), document, catalog(), Instant.parse("2026-08-25T00:00:01Z")) + .planDocument()).path("flows").get(0); + + assertThat(flow.path("container").asText()).isEqualTo("SELL"); + assertThat(flow.path("instrumentIds")).extracting(JsonNode::asText) + .containsExactly(AAPL.toString(), MSFT.toString()); + assertThat(flow.path("steps").get(0).path("parameters")) + .isEqualTo(JSON.readTree("{\"operator\":\"GTE\",\"resolution\":\"1h\",\"thresholdPercent\":\"10\"}")); + assertThat(flow.path("steps").get(1).path("parameters")) + .isEqualTo(JSON.readTree("{\"operator\":\"LT\",\"resolution\":\"1h\",\"thresholdPercent\":\"5\"}")); + } + @Test void sortsInstrumentSpecificFlowsAndCarriesAllocationGroupAndCap() throws Exception { String semantic = """ @@ -53,6 +87,7 @@ void sortsInstrumentSpecificFlowsAndCarriesAllocationGroupAndCap() throws Except } private static BasicStrategyCatalog catalog() { + String conditionSchema = "{\"type\":\"object\",\"properties\":{\"resolution\":{\"type\":\"string\"},\"operator\":{\"type\":\"string\"},\"thresholdPercent\":{\"type\":\"string\"}}}"; String conditionContract = "{\"terminal\":false,\"containers\":[\"BUY\",\"SELL\"],\"runtime\":{\"operation\":\"TEST\",\"arguments\":{}},\"backtest\":{\"supported\":true,\"feeds\":[],\"features\":[]}}"; String orderSchema = "{\"type\":\"object\",\"required\":[\"orderPercent\",\"maxPositionPercent\",\"executionMode\",\"waitMode\",\"waitInterval\",\"maxExecutions\"],\"properties\":{\"orderPercent\":{\"type\":\"string\"},\"maxPositionPercent\":{\"type\":\"string\"},\"executionMode\":{\"type\":\"string\"},\"waitMode\":{\"type\":\"string\"},\"waitInterval\":{\"type\":\"string\"},\"maxExecutions\":{\"type\":\"string\"}}}"; String orderContract = "{\"terminal\":true,\"containers\":[\"BUY\",\"SELL\"],\"runtime\":{\"operation\":\"EMIT_ORDER_CANDIDATE\",\"arguments\":{\"side\":\"$container\",\"orderPercent\":\"$orderPercent\",\"maxPositionPercent\":\"$maxPositionPercent\",\"executionMode\":\"$executionMode\",\"waitMode\":\"$waitMode\",\"waitInterval\":\"$waitInterval\",\"maxExecutions\":\"$maxExecutions\"}},\"backtest\":{\"supported\":true,\"feeds\":[],\"features\":[]}}"; @@ -61,7 +96,7 @@ private static BasicStrategyCatalog catalog() { "basic-elements:2026-08-25", "alpaca-sip/v1", "a".repeat(64), Instant.parse("2026-08-25T00:00:00Z"), null), List.of( - element("TEST_CONDITION", "CONDITION", "{}", conditionContract), + element("TEST_CONDITION", "CONDITION", conditionSchema, conditionContract), element("BASIC_EQUAL_ALLOCATION_ORDER", "ACTION", orderSchema, orderContract)), List.of(), List.of( diff --git a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicStrategyWarningAnalyzerTest.java b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicStrategyWarningAnalyzerTest.java index 384917d7..f009d591 100644 --- a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicStrategyWarningAnalyzerTest.java +++ b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicStrategyWarningAnalyzerTest.java @@ -18,6 +18,19 @@ class BasicStrategyWarningAnalyzerTest { private static final UUID INSTRUMENT_ID = UUID.fromString("25082500-0000-4000-8000-000000000002"); + @Test + void materializedOpposingOccurrencesEmitContradictoryCondition() { + var assembly = assembly(TradeContainer.SELL, List.of( + condition("lower-bound", "BASIC_DRAWDOWN_FROM_PEAK", "GTE", "10"), + condition("upper-bound", "BASIC_DRAWDOWN_FROM_PEAK", "LT", "5")), "1"); + + var warnings = new BasicStrategyWarningAnalyzer().analyze(assembly); + + assertThat(warnings).extracting(StrategyValidationFinding::code) + .contains("CONTRADICTORY_CONDITION") + .doesNotContain("DUPLICATE_CONDITION"); + } + @Test void warnsAboutDuplicateContradictoryAndRepeatedSellExposure() { var assembly = assembly(TradeContainer.SELL, List.of( From 96864119f2b3318d8c7600960de0e41064bfb386 Mon Sep 17 00:00:00 2001 From: Ryan Min Date: Tue, 1 Sep 2026 11:00:53 +0900 Subject: [PATCH 02/13] test: verify final basic runtime arguments --- .../BasicExecutionPlanCompletionTest.java | 51 ++++++++++++++----- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicExecutionPlanCompletionTest.java b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicExecutionPlanCompletionTest.java index 26ee4ca9..dda37fa5 100644 --- a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicExecutionPlanCompletionTest.java +++ b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicExecutionPlanCompletionTest.java @@ -5,9 +5,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.idea2strategy.backend.domain.strategy.ElementCatalogVersion; +import com.idea2strategy.backend.domain.strategy.ImmutableStrategyRelease.Flow; import com.idea2strategy.backend.domain.strategy.StrategyDocument; import com.idea2strategy.backend.domain.strategy.StrategyElementDefinition; import com.idea2strategy.backend.domain.strategy.SupportedInstrument; +import java.math.BigDecimal; import java.time.Instant; import java.util.List; import java.util.UUID; @@ -26,10 +28,12 @@ void preservesOccurrenceArgumentsSideResolutionAndInstrumentSelection() throws E {"id":"sell:bounds","allocationGroupId":"sell","container":"SELL","evaluationMode":"INDEPENDENT","allocationMode":"EQUAL", "instrumentIds":["20000000-0000-4000-8000-000000000001","10000000-0000-4000-8000-000000000001"], "blocks":[ - {"id":"lower","elementCode":"TEST_CONDITION","parameters":{"resolution":"1h","operator":"GTE","thresholdPercent":"10"}}, - {"id":"upper","elementCode":"TEST_CONDITION","parameters":{"resolution":"1h","operator":"LT","thresholdPercent":"5"}}, + {"id":"clock","elementCode":"TEST_CLOCK","parameters":{"resolution":"1h","operator":"GT","reference":"PREVIOUS_CLOSE"}}, + {"id":"lower","elementCode":"TEST_BOUND","parameters":{"operator":"GTE","thresholdPercent":"10"}}, + {"id":"upper","elementCode":"TEST_BOUND","parameters":{"operator":"LT","thresholdPercent":"5"}}, {"id":"order","elementCode":"BASIC_EQUAL_ALLOCATION_ORDER","parameters":{"orderPercent":"25","maxPositionPercent":"40","executionMode":"1회만","waitMode":"조건 재충족","waitInterval":"1","maxExecutions":"1"}}], "connections":[ + {"fromBlockId":"clock","outputPort":"passed","toBlockId":"lower","inputPort":"passed"}, {"fromBlockId":"lower","outputPort":"passed","toBlockId":"upper","inputPort":"passed"}, {"fromBlockId":"upper","outputPort":"passed","toBlockId":"order","inputPort":"passed"}]} ]} @@ -40,17 +44,28 @@ void preservesOccurrenceArgumentsSideResolutionAndInstrumentSelection() throws E StrategyDocumentJson.sha256(canonical), StrategyDocumentJson.sha256("{}"), 1, Instant.parse("2026-08-25T00:00:00Z"), Instant.parse("2026-08-25T00:00:00Z")); - JsonNode flow = JSON.readTree(new BasicExecutionPlanCompiler().compile( - UUID.randomUUID(), document, catalog(), Instant.parse("2026-08-25T00:00:01Z")) - .planDocument()).path("flows").get(0); + JsonNode intermediate = JSON.readTree(new BasicExecutionPlanCompiler().compile( + UUID.randomUUID(), document, catalog(), Instant.parse("2026-08-25T00:00:01Z")) + .planDocument()); + String hash = "a".repeat(64); + Flow releasedFlow = new Flow( + UUID.randomUUID(), "sell:bounds", CATALOG_ID, UUID.randomUUID(), canonical, "{}", + hash, hash, hash, List.of(AAPL, MSFT), List.of(), 0); + JsonNode finalPlan = JSON.readTree(new StrategyBotCompiledPlanAssembler().assemble( + intermediate, catalog(), UUID.randomUUID(), 10000, new BigDecimal("100000"), + List.of(releasedFlow), hash, hash, "basic-launch-snapshot.v1", + Instant.parse("2026-08-25T00:00:02Z")).planDocument()); + JsonNode flow = finalPlan.path("executionSnapshot").path("partitions").get(0).path("flows").get(0); - assertThat(flow.path("container").asText()).isEqualTo("SELL"); - assertThat(flow.path("instrumentIds")).extracting(JsonNode::asText) + assertThat(flow.path("steps").get(0).path("arguments")) + .isEqualTo(JSON.readTree("{\"operator\":\"GT\",\"reference\":\"PREVIOUS_CLOSE\",\"resolution\":\"1h\"}")); + assertThat(flow.path("steps").get(1).path("arguments")) + .isEqualTo(JSON.readTree("{\"operator\":\"GTE\",\"thresholdPercent\":\"10\"}")); + assertThat(flow.path("steps").get(2).path("arguments")) + .isEqualTo(JSON.readTree("{\"operator\":\"LT\",\"thresholdPercent\":\"5\"}")); + assertThat(flow.path("steps").get(3).path("arguments").path("side").asText()).isEqualTo("SELL"); + assertThat(flow.path("officialInstrumentIds")).extracting(JsonNode::asText) .containsExactly(AAPL.toString(), MSFT.toString()); - assertThat(flow.path("steps").get(0).path("parameters")) - .isEqualTo(JSON.readTree("{\"operator\":\"GTE\",\"resolution\":\"1h\",\"thresholdPercent\":\"10\"}")); - assertThat(flow.path("steps").get(1).path("parameters")) - .isEqualTo(JSON.readTree("{\"operator\":\"LT\",\"resolution\":\"1h\",\"thresholdPercent\":\"5\"}")); } @Test @@ -87,8 +102,10 @@ void sortsInstrumentSpecificFlowsAndCarriesAllocationGroupAndCap() throws Except } private static BasicStrategyCatalog catalog() { - String conditionSchema = "{\"type\":\"object\",\"properties\":{\"resolution\":{\"type\":\"string\"},\"operator\":{\"type\":\"string\"},\"thresholdPercent\":{\"type\":\"string\"}}}"; - String conditionContract = "{\"terminal\":false,\"containers\":[\"BUY\",\"SELL\"],\"runtime\":{\"operation\":\"TEST\",\"arguments\":{}},\"backtest\":{\"supported\":true,\"feeds\":[],\"features\":[]}}"; + String clockSchema = "{\"type\":\"object\",\"required\":[\"resolution\",\"operator\",\"reference\"],\"properties\":{\"resolution\":{\"type\":\"string\"},\"operator\":{\"type\":\"string\"},\"reference\":{\"type\":\"string\"}}}"; + String clockContract = "{\"terminal\":false,\"containers\":[\"BUY\",\"SELL\"],\"runtime\":{\"operation\":\"PRICE_COMPARE\",\"arguments\":{\"resolution\":\"$resolution\",\"operator\":\"$operator\",\"reference\":\"$reference\"}},\"backtest\":{\"supported\":true,\"feeds\":[],\"features\":[]}}"; + String boundSchema = "{\"type\":\"object\",\"required\":[\"operator\",\"thresholdPercent\"],\"properties\":{\"operator\":{\"type\":\"string\"},\"thresholdPercent\":{\"type\":\"string\"}}}"; + String boundContract = "{\"terminal\":false,\"containers\":[\"BUY\",\"SELL\"],\"runtime\":{\"operation\":\"DRAWDOWN_FROM_PEAK\",\"arguments\":{\"operator\":\"$operator\",\"thresholdPercent\":\"$thresholdPercent\"}},\"backtest\":{\"supported\":true,\"feeds\":[],\"features\":[]}}"; String orderSchema = "{\"type\":\"object\",\"required\":[\"orderPercent\",\"maxPositionPercent\",\"executionMode\",\"waitMode\",\"waitInterval\",\"maxExecutions\"],\"properties\":{\"orderPercent\":{\"type\":\"string\"},\"maxPositionPercent\":{\"type\":\"string\"},\"executionMode\":{\"type\":\"string\"},\"waitMode\":{\"type\":\"string\"},\"waitInterval\":{\"type\":\"string\"},\"maxExecutions\":{\"type\":\"string\"}}}"; String orderContract = "{\"terminal\":true,\"containers\":[\"BUY\",\"SELL\"],\"runtime\":{\"operation\":\"EMIT_ORDER_CANDIDATE\",\"arguments\":{\"side\":\"$container\",\"orderPercent\":\"$orderPercent\",\"maxPositionPercent\":\"$maxPositionPercent\",\"executionMode\":\"$executionMode\",\"waitMode\":\"$waitMode\",\"waitInterval\":\"$waitInterval\",\"maxExecutions\":\"$maxExecutions\"}},\"backtest\":{\"supported\":true,\"feeds\":[],\"features\":[]}}"; return new BasicStrategyCatalog( @@ -96,7 +113,13 @@ private static BasicStrategyCatalog catalog() { "basic-elements:2026-08-25", "alpaca-sip/v1", "a".repeat(64), Instant.parse("2026-08-25T00:00:00Z"), null), List.of( - element("TEST_CONDITION", "CONDITION", conditionSchema, conditionContract), + element("TEST_CLOCK", "CONDITION", clockSchema, clockContract), + element("TEST_BOUND", "CONDITION", boundSchema, boundContract), + element( + "TEST_CONDITION", + "CONDITION", + "{\"type\":\"object\",\"properties\":{}}", + "{\"terminal\":false,\"containers\":[\"BUY\",\"SELL\"],\"runtime\":{\"operation\":\"TEST\",\"arguments\":{}},\"backtest\":{\"supported\":true,\"feeds\":[],\"features\":[]}}"), element("BASIC_EQUAL_ALLOCATION_ORDER", "ACTION", orderSchema, orderContract)), List.of(), List.of( From 743ff583bec9b6cdef39de62fc7a605f38ae6a0d Mon Sep 17 00:00:00 2001 From: Ryan Min Date: Tue, 1 Sep 2026 12:02:00 +0900 Subject: [PATCH 03/13] test: export basic strategy artifacts through production compiler --- .../BasicStrategyArtifactExporter.java | 150 +++++++++++++ .../StrategyBotCompiledPlanAssembler.java | 139 +++++++++--- .../BasicExecutionPlanCompletionTest.java | 137 ------------ .../StrategyBotCompiledPlanAssemblerTest.java | 87 +++++++- ...actExporterPersistenceIntegrationTest.java | 209 ++++++++++++++++++ 5 files changed, 555 insertions(+), 167 deletions(-) create mode 100644 modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/BasicStrategyArtifactExporter.java delete mode 100644 modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicExecutionPlanCompletionTest.java create mode 100644 modules/backend-persistence/src/test/java/com/idea2strategy/backend/application/strategy/BasicStrategyArtifactExporterPersistenceIntegrationTest.java diff --git a/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/BasicStrategyArtifactExporter.java b/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/BasicStrategyArtifactExporter.java new file mode 100644 index 00000000..c24e369e --- /dev/null +++ b/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/BasicStrategyArtifactExporter.java @@ -0,0 +1,150 @@ +package com.idea2strategy.backend.application.strategy; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.idea2strategy.backend.domain.strategy.ImmutableStrategyRelease.ContractPlan; +import com.idea2strategy.backend.domain.strategy.ImmutableStrategyRelease.Flow; +import com.idea2strategy.backend.domain.strategy.StrategyDocument; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.UUID; + +/** + * Read-only production export boundary for compatibility and deployment verification. + * + *

It runs semantic documents through the same validator/compiler and final contract assembler as + * an immutable release, but does not persist a release, plan, or input pin. This keeps cross-runtime + * verification attached to the producer implementation instead of maintaining a second compiler in + * integration scripts. + */ +public final class BasicStrategyArtifactExporter { + private static final String SNAPSHOT_SCHEMA_VERSION = "basic-launch-snapshot.v1"; + private final ObjectMapper objectMapper = new ObjectMapper() + .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + private final BasicExecutionPlanCompiler compiler = new BasicExecutionPlanCompiler(); + private final StrategyBotCompiledPlanAssembler assembler = new StrategyBotCompiledPlanAssembler(); + + public record PartitionSource(UUID partitionId, int budgetCapBps, String semanticDocument) { + public PartitionSource { + Objects.requireNonNull(partitionId, "partitionId"); + semanticDocument = StrategyDocumentJson.canonicalize(semanticDocument); + } + } + + public ContractPlan export( + List sources, + BasicStrategyCatalog catalog, + BigDecimal initialCashAmount, + Instant exportedAt) { + sources = List.copyOf(Objects.requireNonNull(sources, "sources")); + if (sources.isEmpty()) { + throw new IllegalArgumentException("export must contain at least one partition"); + } + Objects.requireNonNull(catalog, "catalog"); + Objects.requireNonNull(initialCashAmount, "initialCashAmount"); + Objects.requireNonNull(exportedAt, "exportedAt"); + + List partitions = new ArrayList<>(); + for (int index = 0; index < sources.size(); index++) { + PartitionSource source = sources.get(index); + UUID strategyId = derivedId(source.partitionId(), "strategy"); + String presentation = "{}"; + StrategyDocument document = new StrategyDocument( + strategyId, + source.semanticDocument(), + presentation, + "basic-semantic/v1", + "basic-presentation/v1", + StrategyDocumentJson.sha256(source.semanticDocument()), + StrategyDocumentJson.sha256(presentation), + 0, + exportedAt, + exportedAt); + UUID planId = derivedId(source.partitionId(), "compiled-plan"); + JsonNode planRoot = parse(compiler.compile(planId, document, catalog, exportedAt).planDocument()); + partitions.add(new StrategyBotCompiledPlanAssembler.PartitionPlan( + planRoot, + source.partitionId(), + source.budgetCapBps(), + flows(planRoot, planId, catalog.version().id(), exportedAt, index))); + } + + String semanticHash = aggregateSemanticHash(sources); + String snapshotHash = StrategyDocumentJson.sha256(StrategyDocumentJson.canonicalize( + "{\"semanticHash\":\"" + semanticHash + "\",\"snapshotSchemaVersion\":\"" + + SNAPSHOT_SCHEMA_VERSION + "\"}")); + return assembler.assemble( + partitions, + catalog, + initialCashAmount, + semanticHash, + snapshotHash, + SNAPSHOT_SCHEMA_VERSION, + exportedAt); + } + + private List flows( + JsonNode planRoot, + UUID planId, + UUID catalogId, + Instant exportedAt, + int partitionOrder) { + List flows = new ArrayList<>(); + int flowOrder = 0; + for (JsonNode flowNode : planRoot.path("flows")) { + String key = flowNode.path("key").asText(); + List instruments = new ArrayList<>(); + flowNode.path("instrumentIds").forEach(node -> instruments.add(UUID.fromString(node.asText()))); + String semantic = canonical(flowNode); + String layout = "{}"; + String configurationHash = StrategyDocumentJson.sha256( + "export:" + exportedAt + ":" + partitionOrder); + flows.add(new Flow( + derivedId(planId, "flow:" + key), + key, + catalogId, + planId, + semantic, + layout, + StrategyDocumentJson.sha256(semantic), + StrategyDocumentJson.sha256(layout), + configurationHash, + instruments, + List.of(), + flowOrder++)); + } + return List.copyOf(flows); + } + + private String aggregateSemanticHash(List sources) { + var array = objectMapper.createArrayNode(); + sources.forEach(source -> array.add(parse(source.semanticDocument()))); + return StrategyDocumentJson.sha256(canonical(array)); + } + + private JsonNode parse(String document) { + try { + return objectMapper.readTree(document); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("export document is not valid JSON", exception); + } + } + + private String canonical(JsonNode node) { + try { + return StrategyDocumentJson.canonicalize(objectMapper.writeValueAsString(node)); + } catch (JsonProcessingException exception) { + throw new IllegalStateException("export document cannot be serialized", exception); + } + } + + private static UUID derivedId(UUID baseId, String component) { + return UUID.nameUUIDFromBytes((baseId + ":" + component).getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssembler.java b/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssembler.java index e47e5569..59ad4bae 100644 --- a/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssembler.java +++ b/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssembler.java @@ -120,6 +120,21 @@ public final class StrategyBotCompiledPlanAssembler { */ private static final int MONEY_SCALE = 8; + public record PartitionPlan( + JsonNode planRoot, + UUID partitionId, + int budgetCapBps, + List flows) { + public PartitionPlan { + Objects.requireNonNull(planRoot, "planRoot"); + Objects.requireNonNull(partitionId, "partitionId"); + flows = List.copyOf(Objects.requireNonNull(flows, "flows")); + if (flows.isEmpty()) { + throw new IllegalArgumentException("partition must contain at least one flow"); + } + } + } + private final ObjectMapper objectMapper = new ObjectMapper() .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); @@ -143,13 +158,41 @@ public ContractPlan assemble( String snapshotHash, String snapshotSchemaVersion, Instant releasedAt) { - Objects.requireNonNull(planRoot, "planRoot"); + return assemble( + List.of(new PartitionPlan(planRoot, partitionId, budgetCapBps, flows)), + catalog, + initialCashAmount, + semanticHash, + snapshotHash, + snapshotSchemaVersion, + releasedAt); + } + + public ContractPlan assemble( + List partitionPlans, + BasicStrategyCatalog catalog, + BigDecimal initialCashAmount, + String semanticHash, + String snapshotHash, + String snapshotSchemaVersion, + Instant releasedAt) { + partitionPlans = List.copyOf(Objects.requireNonNull(partitionPlans, "partitionPlans")); + if (partitionPlans.isEmpty()) { + throw new IllegalArgumentException("compiled plan must contain at least one partition"); + } Objects.requireNonNull(catalog, "catalog"); - Objects.requireNonNull(partitionId, "partitionId"); Objects.requireNonNull(initialCashAmount, "initialCashAmount"); - Objects.requireNonNull(flows, "flows"); Objects.requireNonNull(releasedAt, "releasedAt"); + JsonNode firstPlanRoot = partitionPlans.getFirst().planRoot(); + String compilerVersion = requiredText(firstPlanRoot, "compilerVersion"); + partitionPlans.forEach(partition -> { + if (!compilerVersion.equals(requiredText(partition.planRoot(), "compilerVersion"))) { + throw new IllegalStateException("all partitions must use one compiler version"); + } + }); + String requiredFeatureSetHash = requiredFeatureSetHash(partitionPlans); + Map elements = new HashMap<>(); catalog.elements().forEach(element -> elements.put(element.elementCode(), element)); Map features = new HashMap<>(); @@ -158,16 +201,16 @@ public ContractPlan assemble( feature.featureCode(), normalizedResolution(feature.resolution())), feature)); - Map> containers = containersByKey(planRoot, elements); - List requiredFeatures = requiredFeatures(planRoot, elements, features); + List requiredFeatures = requiredFeatures( + partitionPlans.stream().map(PartitionPlan::planRoot).toList(), elements, features); ObjectNode root = objectMapper.createObjectNode(); root.put("contractVersion", CONTRACT_VERSION); root.put("schemaVersion", PLAN_SCHEMA_VERSION); root.put("elementCatalogVersion", catalog.version().catalogVersion()); root.put("instrumentCatalogVersion", PUBLISHED_INSTRUMENT_CATALOG_VERSION); - root.put("compilerVersion", requiredText(planRoot, "compilerVersion")); - root.put("requiredFeatureSetHash", prefixed(requiredText(planRoot, "requiredFeatureSetHash"))); + root.put("compilerVersion", compilerVersion); + root.put("requiredFeatureSetHash", prefixed(requiredFeatureSetHash)); ArrayNode featureNodes = root.putArray("requiredFeatures"); requiredFeatures.forEach(feature -> { @@ -190,24 +233,27 @@ public ContractPlan assemble( snapshot.put("initialCashAmount", moneyAmount(initialCashAmount)); snapshot.put("currency", "USD"); ArrayNode partitions = snapshot.putArray("partitions"); - ObjectNode partition = partitions.addObject(); - partition.put("key", partitionId.toString()); - partition.put("budgetCapBps", budgetCapBps); - ArrayNode flowNodes = partition.putArray("flows"); - flows.stream() - .sorted(java.util.Comparator.comparingInt(Flow::positionOrder)) - .forEach(flow -> { - ObjectNode node = flowNodes.addObject(); - node.put("key", flow.name()); - ArrayNode instruments = node.putArray("officialInstrumentIds"); - flow.instrumentIds().stream().map(UUID::toString).sorted().forEach(instruments::add); - List container = containers.get(flow.name()); - if (container == null) { - throw new IllegalStateException( - "released flow " + flow.name() + " has no compiled container: the " - + "release and the compiled plan disagree about which flows exist"); - } - appendSteps(node.putArray("steps"), container); + partitionPlans.forEach(partitionPlan -> { + Map> containers = containersByKey(partitionPlan.planRoot(), elements); + ObjectNode partition = partitions.addObject(); + partition.put("key", partitionPlan.partitionId().toString()); + partition.put("budgetCapBps", partitionPlan.budgetCapBps()); + ArrayNode flowNodes = partition.putArray("flows"); + partitionPlan.flows().stream() + .sorted(java.util.Comparator.comparingInt(Flow::positionOrder)) + .forEach(flow -> { + ObjectNode node = flowNodes.addObject(); + node.put("key", flow.name()); + ArrayNode instruments = node.putArray("officialInstrumentIds"); + flow.instrumentIds().stream().map(UUID::toString).sorted().forEach(instruments::add); + List container = containers.get(flow.name()); + if (container == null) { + throw new IllegalStateException( + "released flow " + flow.name() + " has no compiled container: the " + + "release and the compiled plan disagree about which flows exist"); + } + appendSteps(node.putArray("steps"), container); + }); }); String checksum = checksum(root, requiredFeatures); @@ -331,11 +377,13 @@ private String resolve( * instruments requiring the same feature would produce exactly that. */ private List requiredFeatures( - JsonNode planRoot, + List planRoots, Map elements, Map features) { Map> instrumentsByFeature = new LinkedHashMap<>(); - for (JsonNode flow : planRoot.path("flows")) { + for (JsonNode flow : planRoots.stream() + .flatMap(planRoot -> planRoot.path("flows").valueStream()) + .toList()) { Set instruments = new LinkedHashSet<>(); flow.path("instrumentIds").forEach(node -> instruments.add(node.asText())); Set requirements = new LinkedHashSet<>(); @@ -382,6 +430,43 @@ private List requiredFeatures( return List.copyOf(requirements); } + /** + * The partition compiler identifies its canonical feature-definition document. A plan with more + * than one distinct partition-local document publishes the identity of their sorted union, + * matching the compiler's existing feature-document algorithm. The common single-document path + * deliberately preserves the compiler's emitted hash byte-for-byte. + */ + private String requiredFeatureSetHash(List partitionPlans) { + Set partitionHashes = partitionPlans.stream() + .map(partition -> requiredText(partition.planRoot(), "requiredFeatureSetHash")) + .collect(java.util.stream.Collectors.toCollection(LinkedHashSet::new)); + if (partitionHashes.size() == 1) { + return partitionHashes.iterator().next(); + } + + Map featureDocuments = new java.util.TreeMap<>(); + for (PartitionPlan partition : partitionPlans) { + JsonNode features = partition.planRoot().path("requiredFeatures"); + if (!features.isArray()) { + throw new IllegalStateException( + "A partition with a distinct feature set must carry requiredFeatures"); + } + features.forEach(feature -> { + String key = requiredText(feature, "featureCode") + "\u0000" + + requiredText(feature, "resolution"); + String document = canonical(feature); + String previous = featureDocuments.putIfAbsent(key, document); + if (previous != null && !previous.equals(document)) { + throw new IllegalStateException( + "Partitions disagree about required feature " + key.replace('\u0000', '@')); + } + }); + } + ArrayNode union = objectMapper.createArrayNode(); + featureDocuments.values().forEach(document -> union.add(parse(document))); + return StrategyDocumentJson.sha256(canonical(union)); + } + private FeatureRequirementKey resolveFeatureRequirement( Map features, String featureCode, diff --git a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicExecutionPlanCompletionTest.java b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicExecutionPlanCompletionTest.java deleted file mode 100644 index dda37fa5..00000000 --- a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/BasicExecutionPlanCompletionTest.java +++ /dev/null @@ -1,137 +0,0 @@ -package com.idea2strategy.backend.application.strategy; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.idea2strategy.backend.domain.strategy.ElementCatalogVersion; -import com.idea2strategy.backend.domain.strategy.ImmutableStrategyRelease.Flow; -import com.idea2strategy.backend.domain.strategy.StrategyDocument; -import com.idea2strategy.backend.domain.strategy.StrategyElementDefinition; -import com.idea2strategy.backend.domain.strategy.SupportedInstrument; -import java.math.BigDecimal; -import java.time.Instant; -import java.util.List; -import java.util.UUID; -import org.junit.jupiter.api.Test; - -class BasicExecutionPlanCompletionTest { - private static final ObjectMapper JSON = new ObjectMapper(); - private static final UUID CATALOG_ID = UUID.fromString("0f5a0000-0000-4000-8000-000000000001"); - private static final UUID AAPL = UUID.fromString("10000000-0000-4000-8000-000000000001"); - private static final UUID MSFT = UUID.fromString("20000000-0000-4000-8000-000000000001"); - - @Test - void preservesOccurrenceArgumentsSideResolutionAndInstrumentSelection() throws Exception { - String semantic = """ - {"catalogId":"0f5a0000-0000-4000-8000-000000000001","groups":[ - {"id":"sell:bounds","allocationGroupId":"sell","container":"SELL","evaluationMode":"INDEPENDENT","allocationMode":"EQUAL", - "instrumentIds":["20000000-0000-4000-8000-000000000001","10000000-0000-4000-8000-000000000001"], - "blocks":[ - {"id":"clock","elementCode":"TEST_CLOCK","parameters":{"resolution":"1h","operator":"GT","reference":"PREVIOUS_CLOSE"}}, - {"id":"lower","elementCode":"TEST_BOUND","parameters":{"operator":"GTE","thresholdPercent":"10"}}, - {"id":"upper","elementCode":"TEST_BOUND","parameters":{"operator":"LT","thresholdPercent":"5"}}, - {"id":"order","elementCode":"BASIC_EQUAL_ALLOCATION_ORDER","parameters":{"orderPercent":"25","maxPositionPercent":"40","executionMode":"1회만","waitMode":"조건 재충족","waitInterval":"1","maxExecutions":"1"}}], - "connections":[ - {"fromBlockId":"clock","outputPort":"passed","toBlockId":"lower","inputPort":"passed"}, - {"fromBlockId":"lower","outputPort":"passed","toBlockId":"upper","inputPort":"passed"}, - {"fromBlockId":"upper","outputPort":"passed","toBlockId":"order","inputPort":"passed"}]} - ]} - """; - String canonical = StrategyDocumentJson.canonicalize(semantic); - var document = new StrategyDocument( - UUID.randomUUID(), canonical, "{}", "basic-semantic/v1", "basic-presentation/v1", - StrategyDocumentJson.sha256(canonical), StrategyDocumentJson.sha256("{}"), 1, - Instant.parse("2026-08-25T00:00:00Z"), Instant.parse("2026-08-25T00:00:00Z")); - - JsonNode intermediate = JSON.readTree(new BasicExecutionPlanCompiler().compile( - UUID.randomUUID(), document, catalog(), Instant.parse("2026-08-25T00:00:01Z")) - .planDocument()); - String hash = "a".repeat(64); - Flow releasedFlow = new Flow( - UUID.randomUUID(), "sell:bounds", CATALOG_ID, UUID.randomUUID(), canonical, "{}", - hash, hash, hash, List.of(AAPL, MSFT), List.of(), 0); - JsonNode finalPlan = JSON.readTree(new StrategyBotCompiledPlanAssembler().assemble( - intermediate, catalog(), UUID.randomUUID(), 10000, new BigDecimal("100000"), - List.of(releasedFlow), hash, hash, "basic-launch-snapshot.v1", - Instant.parse("2026-08-25T00:00:02Z")).planDocument()); - JsonNode flow = finalPlan.path("executionSnapshot").path("partitions").get(0).path("flows").get(0); - - assertThat(flow.path("steps").get(0).path("arguments")) - .isEqualTo(JSON.readTree("{\"operator\":\"GT\",\"reference\":\"PREVIOUS_CLOSE\",\"resolution\":\"1h\"}")); - assertThat(flow.path("steps").get(1).path("arguments")) - .isEqualTo(JSON.readTree("{\"operator\":\"GTE\",\"thresholdPercent\":\"10\"}")); - assertThat(flow.path("steps").get(2).path("arguments")) - .isEqualTo(JSON.readTree("{\"operator\":\"LT\",\"thresholdPercent\":\"5\"}")); - assertThat(flow.path("steps").get(3).path("arguments").path("side").asText()).isEqualTo("SELL"); - assertThat(flow.path("officialInstrumentIds")).extracting(JsonNode::asText) - .containsExactly(AAPL.toString(), MSFT.toString()); - } - - @Test - void sortsInstrumentSpecificFlowsAndCarriesAllocationGroupAndCap() throws Exception { - String semantic = """ - {"catalogId":"0f5a0000-0000-4000-8000-000000000001","groups":[ - {"id":"buy:msft","allocationGroupId":"buy","container":"BUY","evaluationMode":"INDEPENDENT","allocationMode":"EQUAL", - "instrumentIds":["20000000-0000-4000-8000-000000000001"], - "blocks":[{"id":"condition","elementCode":"TEST_CONDITION","parameters":{}},{"id":"order","elementCode":"BASIC_EQUAL_ALLOCATION_ORDER","parameters":{"orderPercent":"25","maxPositionPercent":"40","executionMode":"1회만","waitMode":"조건 재충족","waitInterval":"1","maxExecutions":"1"}}], - "connections":[{"fromBlockId":"condition","outputPort":"passed","toBlockId":"order","inputPort":"passed"}]}, - {"id":"buy:aapl","allocationGroupId":"buy","container":"BUY","evaluationMode":"INDEPENDENT","allocationMode":"EQUAL", - "instrumentIds":["10000000-0000-4000-8000-000000000001"], - "blocks":[{"id":"condition","elementCode":"TEST_CONDITION","parameters":{}},{"id":"order","elementCode":"BASIC_EQUAL_ALLOCATION_ORDER","parameters":{"orderPercent":"25","maxPositionPercent":"25","executionMode":"1회만","waitMode":"조건 재충족","waitInterval":"1","maxExecutions":"1"}}], - "connections":[{"fromBlockId":"condition","outputPort":"passed","toBlockId":"order","inputPort":"passed"}]} - ]} - """; - String canonical = StrategyDocumentJson.canonicalize(semantic); - var document = new StrategyDocument( - UUID.randomUUID(), canonical, "{}", "basic-semantic/v1", "basic-presentation/v1", - StrategyDocumentJson.sha256(canonical), StrategyDocumentJson.sha256("{}"), 1, - Instant.parse("2026-08-25T00:00:00Z"), Instant.parse("2026-08-25T00:00:00Z")); - - var plan = new BasicExecutionPlanCompiler().compile( - UUID.randomUUID(), document, catalog(), Instant.parse("2026-08-25T00:00:01Z")); - JsonNode flows = JSON.readTree(plan.planDocument()).path("flows"); - - assertThat(flows).extracting(flow -> flow.path("key").asText()) - .containsExactly("buy:aapl", "buy:msft"); - assertThat(flows).allSatisfy(flow -> assertThat(flow.path("allocationGroupId").asText()).isEqualTo("buy")); - assertThat(flows.get(0).path("steps").get(1).path("parameters").path("maxPositionPercent").asText()) - .isEqualTo("25"); - assertThat(flows.get(1).path("steps").get(1).path("parameters").path("maxPositionPercent").asText()) - .isEqualTo("40"); - } - - private static BasicStrategyCatalog catalog() { - String clockSchema = "{\"type\":\"object\",\"required\":[\"resolution\",\"operator\",\"reference\"],\"properties\":{\"resolution\":{\"type\":\"string\"},\"operator\":{\"type\":\"string\"},\"reference\":{\"type\":\"string\"}}}"; - String clockContract = "{\"terminal\":false,\"containers\":[\"BUY\",\"SELL\"],\"runtime\":{\"operation\":\"PRICE_COMPARE\",\"arguments\":{\"resolution\":\"$resolution\",\"operator\":\"$operator\",\"reference\":\"$reference\"}},\"backtest\":{\"supported\":true,\"feeds\":[],\"features\":[]}}"; - String boundSchema = "{\"type\":\"object\",\"required\":[\"operator\",\"thresholdPercent\"],\"properties\":{\"operator\":{\"type\":\"string\"},\"thresholdPercent\":{\"type\":\"string\"}}}"; - String boundContract = "{\"terminal\":false,\"containers\":[\"BUY\",\"SELL\"],\"runtime\":{\"operation\":\"DRAWDOWN_FROM_PEAK\",\"arguments\":{\"operator\":\"$operator\",\"thresholdPercent\":\"$thresholdPercent\"}},\"backtest\":{\"supported\":true,\"feeds\":[],\"features\":[]}}"; - String orderSchema = "{\"type\":\"object\",\"required\":[\"orderPercent\",\"maxPositionPercent\",\"executionMode\",\"waitMode\",\"waitInterval\",\"maxExecutions\"],\"properties\":{\"orderPercent\":{\"type\":\"string\"},\"maxPositionPercent\":{\"type\":\"string\"},\"executionMode\":{\"type\":\"string\"},\"waitMode\":{\"type\":\"string\"},\"waitInterval\":{\"type\":\"string\"},\"maxExecutions\":{\"type\":\"string\"}}}"; - String orderContract = "{\"terminal\":true,\"containers\":[\"BUY\",\"SELL\"],\"runtime\":{\"operation\":\"EMIT_ORDER_CANDIDATE\",\"arguments\":{\"side\":\"$container\",\"orderPercent\":\"$orderPercent\",\"maxPositionPercent\":\"$maxPositionPercent\",\"executionMode\":\"$executionMode\",\"waitMode\":\"$waitMode\",\"waitInterval\":\"$waitInterval\",\"maxExecutions\":\"$maxExecutions\"}},\"backtest\":{\"supported\":true,\"feeds\":[],\"features\":[]}}"; - return new BasicStrategyCatalog( - new ElementCatalogVersion(CATALOG_ID, "basic/v1", "basic-semantic/v1", - "basic-elements:2026-08-25", "alpaca-sip/v1", "a".repeat(64), - Instant.parse("2026-08-25T00:00:00Z"), null), - List.of( - element("TEST_CLOCK", "CONDITION", clockSchema, clockContract), - element("TEST_BOUND", "CONDITION", boundSchema, boundContract), - element( - "TEST_CONDITION", - "CONDITION", - "{\"type\":\"object\",\"properties\":{}}", - "{\"terminal\":false,\"containers\":[\"BUY\",\"SELL\"],\"runtime\":{\"operation\":\"TEST\",\"arguments\":{}},\"backtest\":{\"supported\":true,\"feeds\":[],\"features\":[]}}"), - element("BASIC_EQUAL_ALLOCATION_ORDER", "ACTION", orderSchema, orderContract)), - List.of(), - List.of( - new SupportedInstrument(AAPL, "STOCK", "XNAS", "USD", "AAPL"), - new SupportedInstrument(MSFT, "STOCK", "XNAS", "USD", "MSFT"))); - } - - private static StrategyElementDefinition element(String code, String kind, String schema, String contract) { - return new StrategyElementDefinition( - UUID.nameUUIDFromBytes(code.getBytes(java.nio.charset.StandardCharsets.UTF_8)), CATALOG_ID, - code, kind, schema, "{\"passed\":{\"type\":\"boolean\"}}", - kind.equals("ACTION") ? "{}" : "{\"passed\":{\"type\":\"boolean\"}}", - contract, "b".repeat(64)); - } -} diff --git a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssemblerTest.java b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssemblerTest.java index 6432b0b0..4d7c97c5 100644 --- a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssemblerTest.java +++ b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssemblerTest.java @@ -34,6 +34,82 @@ class StrategyBotCompiledPlanAssemblerTest { private final ObjectMapper objectMapper = new ObjectMapper(); private final StrategyBotCompiledPlanAssembler assembler = new StrategyBotCompiledPlanAssembler(); + @Test + void publishesEveryCompiledPartitionUnderItsExactIdentity() { + UUID secondPartition = UUID.fromString("41000000-0000-4000-8000-000000000002"); + JsonNode firstRoot = planWith(buyFlow("buy", List.of(AAPL))); + JsonNode secondRoot = planWith(sellFlow("sell", List.of(MSFT))); + + ContractPlan plan = assembler.assemble( + List.of( + new StrategyBotCompiledPlanAssembler.PartitionPlan( + firstRoot, PARTITION_ID, 10_000, flowRecords(firstRoot)), + new StrategyBotCompiledPlanAssembler.PartitionPlan( + secondRoot, secondPartition, 10_000, flowRecords(secondRoot))), + catalog(feature("RSI_14", "rsi:1.0.0", "30m", 15)), + new BigDecimal("100000.00"), HASH_A, HASH_B, + "basic-launch-snapshot.v1", RELEASED_AT); + + JsonNode partitions = parse(plan.planDocument()).path("executionSnapshot").path("partitions"); + assertThat(partitions).hasSize(2); + assertThat(partitions).extracting(node -> node.path("key").asText()) + .containsExactly(PARTITION_ID.toString(), secondPartition.toString()); + assertThat(sideOf(partitions.get(0).path("flows").get(0))).isEqualTo("BUY"); + assertThat(sideOf(partitions.get(1).path("flows").get(0))).isEqualTo("SELL"); + } + + @Test + void hashesTheUnionWhenPartitionsRequireDifferentFeatureSets() { + UUID secondPartition = UUID.fromString("41000000-0000-4000-8000-000000000002"); + JsonNode directFlow = objectMapper.createObjectNode() + .put("key", "direct") + .put("container", "BUY") + .set( + "instrumentIds", objectMapper.createArrayNode().add(AAPL.toString())) + .set("steps", objectMapper.createArrayNode() + .add(step(1, "TEST_CONDITION", "{}")) + .add(step(2, "BASIC_EQUAL_ALLOCATION_ORDER", "{}"))); + var directRoot = (com.fasterxml.jackson.databind.node.ObjectNode) planWith(directFlow); + directRoot.put("requiredFeatureSetHash", + "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"); + directRoot.set("requiredFeatures", objectMapper.createArrayNode()); + var featureRoot = (com.fasterxml.jackson.databind.node.ObjectNode) + planWith(sellFlow("feature", List.of(MSFT))); + featureRoot.put("requiredFeatureSetHash", + "160d96f04548e15fed4c1e23abb3ccbdd1e6f067ba94b4451a32ce8ca2a2e94f"); + featureRoot.set("requiredFeatures", parse(""" + [{"calculatorVersion":"rsi:1.0.0","definitionHash":"%s", + "featureCode":"RSI_14","normalizedParameters":{"period":14}, + "outputValueType":"NUMBER","requiredHistoryPoints":15,"resolution":"30m"}] + """.formatted(HASH_B))); + BasicStrategyCatalog selected = new BasicStrategyCatalog( + catalog().version(), + List.of( + element("TEST_CONDITION", "TEST", "{}", "[]"), + element("BASIC_RSI_READ", "LOAD_FEATURE", + "{\"feature\":\"RSI_14\",\"resolution\":\"$resolution\"}", + "[\"RSI_14\"]"), + element("BASIC_VALUE_COMPARE", "COMPARE", + "{\"operator\":\"$operator\",\"threshold\":\"$threshold\"}", "[]"), + element("BASIC_EQUAL_ALLOCATION_ORDER", "EMIT_ORDER_CANDIDATE", + "{\"allocation\":\"EQUAL\",\"orderType\":\"MARKET\"," + + "\"side\":\"$container\"}", "[]")), + List.of(feature("RSI_14", "rsi:1.0.0", "30m", 15)), + catalog().instruments()); + + ContractPlan plan = assembler.assemble( + List.of( + new StrategyBotCompiledPlanAssembler.PartitionPlan( + directRoot, PARTITION_ID, 10_000, flowRecords(directRoot)), + new StrategyBotCompiledPlanAssembler.PartitionPlan( + featureRoot, secondPartition, 10_000, flowRecords(featureRoot))), + selected, new BigDecimal("100000.00"), HASH_A, HASH_B, + "basic-launch-snapshot.v1", RELEASED_AT); + + assertThat(parse(plan.planDocument()).path("requiredFeatureSetHash").asText()) + .isEqualTo("sha256:160d96f04548e15fed4c1e23abb3ccbdd1e6f067ba94b4451a32ce8ca2a2e94f"); + } + @Test void publishesTheElementCatalogRuntimeOperationsRatherThanItsElementCodes() { ContractPlan plan = assemble(planWith(buyFlow("buy", List.of(AAPL)))); @@ -293,6 +369,13 @@ private ContractPlan assemble(JsonNode planRoot, StrategyFeatureDefinition featu } private ContractPlan assembleWithCatalog(JsonNode planRoot, BasicStrategyCatalog selectedCatalog) { + List flows = flowRecords(planRoot); + return assembler.assemble( + planRoot, selectedCatalog, PARTITION_ID, 10_000, new BigDecimal("100000.00"), flows, + HASH_A, HASH_B, "basic-launch-snapshot.v1", RELEASED_AT); + } + + private List flowRecords(JsonNode planRoot) { List flows = new java.util.ArrayList<>(); int order = 0; for (JsonNode flowNode : planRoot.path("flows")) { @@ -304,9 +387,7 @@ private ContractPlan assembleWithCatalog(JsonNode planRoot, BasicStrategyCatalog flowNode.path("key").asText(), CATALOG_ID, UUID.randomUUID(), "{}", "{}", HASH_A, HASH_B, HASH_A, instruments, List.of(), order++)); } - return assembler.assemble( - planRoot, selectedCatalog, PARTITION_ID, 10_000, new BigDecimal("100000.00"), flows, - HASH_A, HASH_B, "basic-launch-snapshot.v1", RELEASED_AT); + return List.copyOf(flows); } private JsonNode planWith(JsonNode... flows) { diff --git a/modules/backend-persistence/src/test/java/com/idea2strategy/backend/application/strategy/BasicStrategyArtifactExporterPersistenceIntegrationTest.java b/modules/backend-persistence/src/test/java/com/idea2strategy/backend/application/strategy/BasicStrategyArtifactExporterPersistenceIntegrationTest.java new file mode 100644 index 00000000..225d8c0a --- /dev/null +++ b/modules/backend-persistence/src/test/java/com/idea2strategy/backend/application/strategy/BasicStrategyArtifactExporterPersistenceIntegrationTest.java @@ -0,0 +1,209 @@ +package com.idea2strategy.backend.application.strategy; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.idea2strategy.backend.persistence.strategy.BasicStrategyCatalogJooqQueryAdapter; +import java.math.BigDecimal; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.List; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.transaction.annotation.Transactional; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.postgresql.PostgreSQLContainer; + +@Testcontainers(disabledWithoutDocker = true) +@SpringBootTest(classes = BasicStrategyArtifactExporterPersistenceIntegrationTest.TestApplication.class) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class BasicStrategyArtifactExporterPersistenceIntegrationTest { + private static final ObjectMapper JSON = new ObjectMapper(); + private static final Instant NOW = Instant.parse("2026-08-31T00:00:00Z"); + private static final UUID AAPL = UUID.fromString("03e7e685-d6da-4f1f-9279-91477884aab9"); + private static final UUID MSFT = UUID.fromString("00000000-0000-4000-8000-000000000302"); + private static final List SELECTED_INSTRUMENTS = List.of( + AAPL, + MSFT, + UUID.fromString("00000000-0000-4000-8000-000000000303"), + UUID.fromString("00000000-0000-4000-8000-000000000304"), + UUID.fromString("00000000-0000-4000-8000-000000000305")); + private static final UUID PARTITION = UUID.fromString("71000000-0000-4000-8000-000000000001"); + + @Container + static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer("postgres:16-alpine"); + + @DynamicPropertySource + static void databaseProperties(DynamicPropertyRegistry registry) { + registry.add("spring.datasource.url", POSTGRES::getJdbcUrl); + registry.add("spring.datasource.username", POSTGRES::getUsername); + registry.add("spring.datasource.password", POSTGRES::getPassword); + registry.add("spring.jpa.hibernate.ddl-auto", () -> "none"); + registry.add("spring.flyway.enabled", () -> "true"); + } + + @Autowired + private BasicStrategyCatalogJooqQueryAdapter catalogAdapter; + + @Autowired + private JdbcTemplate jdbc; + + @BeforeEach + void addSelectedInstrumentsToTheMigratedCatalogUniverse() { + for (int index = 0; index < SELECTED_INSTRUMENTS.size(); index++) { + insertInstrument(SELECTED_INSTRUMENTS.get(index), "TASK3-" + (index + 1)); + } + } + + @Test + void exportsFinalArgumentsFromTheOfficialPersistedExecutionContracts() throws Exception { + BasicStrategyCatalog catalog = catalog(); + String semantic = semantic(catalog.version().id()); + + var plan = new BasicStrategyArtifactExporter().export( + List.of(new BasicStrategyArtifactExporter.PartitionSource(PARTITION, 10_000, semantic)), + catalog, + new BigDecimal("100000"), + NOW); + + JsonNode root = JSON.readTree(plan.planDocument()); + JsonNode flow = root.path("executionSnapshot").path("partitions").get(0).path("flows").get(0); + assertThat(root.path("elementCatalogVersion").asText()).isEqualTo("basic-elements:2026-08-25"); + assertThat(flow.path("officialInstrumentIds")).extracting(JsonNode::asText) + .containsExactly(MSFT.toString(), AAPL.toString()); + assertThat(flow.path("steps").get(0).path("arguments")) + .isEqualTo(JSON.readTree("{\"operator\":\"GT\",\"reference\":\"PREVIOUS_CLOSE\",\"resolution\":\"30m\"}")); + assertThat(flow.path("steps").get(1).path("arguments")) + .isEqualTo(JSON.readTree("{\"operator\":\"GTE\",\"thresholdPercent\":\"10\"}")); + assertThat(flow.path("steps").get(2).path("arguments")) + .isEqualTo(JSON.readTree("{\"operator\":\"LT\",\"thresholdPercent\":\"5\"}")); + assertThat(flow.path("steps").get(3).path("arguments")) + .isEqualTo(JSON.readTree("{\"allocation\":\"EQUAL\",\"executionMode\":\"1회만\",\"maxExecutions\":\"1\",\"maxPositionPercent\":\"40\",\"orderPercent\":\"25\",\"orderType\":\"MARKET\",\"side\":\"SELL\",\"timeInForce\":\"DAY\",\"waitInterval\":\"1\",\"waitMode\":\"조건 재충족\"}")); + } + + @Test + @Transactional + void persistedRuntimeMappingIsLoadBearingForTheEmittedArtifact() throws Exception { + jdbc.update(""" + update strategy.element_definitions + set execution_contract = jsonb_set(execution_contract, '{runtime,arguments}', + '{"operator":"$operator","resolution":"$resolution"}'::jsonb) + where element_catalog_version_id = '0f5a0000-0000-4000-8000-000000000001'::uuid + and element_code = 'BASIC_PRICE_COMPARE' + """); + BasicStrategyCatalog catalog = catalog(); + + var plan = new BasicStrategyArtifactExporter().export( + List.of(new BasicStrategyArtifactExporter.PartitionSource( + PARTITION, 10_000, semantic(catalog.version().id()))), + catalog, + new BigDecimal("100000"), + NOW); + + JsonNode clockArguments = JSON.readTree(plan.planDocument()) + .path("executionSnapshot").path("partitions").get(0) + .path("flows").get(0).path("steps").get(0).path("arguments"); + assertThat(clockArguments.has("reference")).isFalse(); + assertThat(clockArguments.path("operator").asText()).isEqualTo("GT"); + assertThat(clockArguments.path("resolution").asText()).isEqualTo("30m"); + } + + @Test + void exportsTheRootCompatibilityBundleThroughTheProductionBoundary() throws Exception { + String inputPath = System.getenv("TASK3_BACKEND_EXPORT_INPUT"); + String outputPath = System.getenv("TASK3_BACKEND_EXPORT_OUTPUT"); + Assumptions.assumeTrue(inputPath != null && outputPath != null); + + JsonNode request = JSON.readTree(Files.readString(Path.of(inputPath))); + var output = JSON.createObjectNode(); + var cases = output.putArray("cases"); + BasicStrategyCatalog catalog = catalog(); + BasicStrategyArtifactExporter exporter = new BasicStrategyArtifactExporter(); + for (JsonNode requestedCase : request.path("cases")) { + List sources = new java.util.ArrayList<>(); + for (JsonNode partition : requestedCase.path("partitions")) { + sources.add(new BasicStrategyArtifactExporter.PartitionSource( + UUID.fromString(partition.path("key").asText()), + partition.path("budgetCapBps").asInt(), + JSON.writeValueAsString(partition.path("semanticDocument")))); + } + com.idea2strategy.backend.domain.strategy.ImmutableStrategyRelease.ContractPlan plan; + try { + plan = exporter.export(sources, catalog, new BigDecimal("100000"), NOW); + } catch (RuntimeException failure) { + throw new IllegalStateException( + "backend export rejected case " + requestedCase.path("name").asText(), failure); + } + var item = cases.addObject(); + item.put("name", requestedCase.path("name").asText()); + item.put("planDocument", plan.planDocument()); + } + Files.writeString(Path.of(outputPath), JSON.writeValueAsString(output)); + } + + private BasicStrategyCatalog catalog() { + return new BasicStrategyCatalogQueryService( + catalogAdapter, + Clock.fixed(NOW, ZoneOffset.UTC), + ZoneId.of("America/New_York")) + .getPublished("basic/v1", "basic-semantic/v1", "basic-elements:2026-08-25"); + } + + private static String semantic(UUID catalogId) { + return StrategyDocumentJson.canonicalize(""" + {"catalogId":"%s","groups":[{ + "id":"sell:bounds","allocationGroupId":"sell","container":"SELL", + "evaluationMode":"INDEPENDENT","allocationMode":"EQUAL", + "instrumentIds":["%s","%s"], + "blocks":[ + {"id":"clock","elementCode":"BASIC_PRICE_COMPARE","parameters":{"resolution":"30m","operator":"GT","reference":"PREVIOUS_CLOSE"}}, + {"id":"lower","elementCode":"BASIC_DRAWDOWN_FROM_PEAK","parameters":{"operator":"GTE","thresholdPercent":"10"}}, + {"id":"upper","elementCode":"BASIC_DRAWDOWN_FROM_PEAK","parameters":{"operator":"LT","thresholdPercent":"5"}}, + {"id":"order","elementCode":"BASIC_EQUAL_ALLOCATION_ORDER","parameters":{"orderPercent":"25","maxPositionPercent":"40","executionMode":"1회만","waitMode":"조건 재충족","waitInterval":"1","maxExecutions":"1"}}], + "connections":[ + {"fromBlockId":"clock","outputPort":"passed","toBlockId":"lower","inputPort":"passed"}, + {"fromBlockId":"lower","outputPort":"passed","toBlockId":"upper","inputPort":"passed"}, + {"fromBlockId":"upper","outputPort":"passed","toBlockId":"order","inputPort":"passed"}] + }]} + """.formatted(catalogId, AAPL, MSFT)); + } + + private void insertInstrument(UUID id, String symbol) { + jdbc.update(""" + insert into market_data.instruments + (id, asset_type, primary_exchange_mic, currency_code, listed_at, created_at) + values (?, 'STOCK', 'XNAS', 'USD', date '2020-01-01', ?) + on conflict (id) do nothing + """, id, NOW.atOffset(ZoneOffset.UTC)); + jdbc.update(""" + insert into market_data.instrument_symbols + (id, instrument_id, exchange_mic, symbol, effective_from) + select ?, ?, 'XNAS', ?, ? + where not exists ( + select 1 from market_data.instrument_symbols where instrument_id = ?) + """, UUID.nameUUIDFromBytes((id + ":symbol").getBytes()), id, symbol, + NOW.minusSeconds(3600).atOffset(ZoneOffset.UTC), id); + } + + @SpringBootConfiguration + @EnableAutoConfiguration + @Import(BasicStrategyCatalogJooqQueryAdapter.class) + static class TestApplication {} +} From ad707f298c9a0f7893e51ab87f60abbfd2e846d2 Mon Sep 17 00:00:00 2001 From: Ryan Min Date: Tue, 1 Sep 2026 12:35:17 +0900 Subject: [PATCH 04/13] fix: validate partition feature-set identities --- .../StrategyBotCompiledPlanAssembler.java | 28 ++-- .../StrategyBotCompiledPlanAssemblerTest.java | 128 +++++++++++++++++- 2 files changed, 143 insertions(+), 13 deletions(-) diff --git a/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssembler.java b/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssembler.java index 59ad4bae..276e8494 100644 --- a/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssembler.java +++ b/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssembler.java @@ -434,12 +434,28 @@ private List requiredFeatures( * The partition compiler identifies its canonical feature-definition document. A plan with more * than one distinct partition-local document publishes the identity of their sorted union, * matching the compiler's existing feature-document algorithm. The common single-document path - * deliberately preserves the compiler's emitted hash byte-for-byte. + * deliberately preserves the compiler's emitted hash byte-for-byte. Every partition's claim is + * re-derived first, so neither the single-document path nor a shared forged hash can bypass the + * feature definitions the compiler artifact actually carries. */ private String requiredFeatureSetHash(List partitionPlans) { - Set partitionHashes = partitionPlans.stream() - .map(partition -> requiredText(partition.planRoot(), "requiredFeatureSetHash")) - .collect(java.util.stream.Collectors.toCollection(LinkedHashSet::new)); + Set partitionHashes = new LinkedHashSet<>(); + for (PartitionPlan partition : partitionPlans) { + JsonNode features = partition.planRoot().path("requiredFeatures"); + if (!features.isArray()) { + throw new IllegalStateException( + "Partition " + partition.partitionId() + + " requiredFeatures must be an array"); + } + String claimed = requiredText(partition.planRoot(), "requiredFeatureSetHash"); + String actual = StrategyDocumentJson.sha256(canonical(features)); + if (!actual.equals(claimed)) { + throw new IllegalStateException( + "Partition " + partition.partitionId() + + " requiredFeatureSetHash does not match requiredFeatures"); + } + partitionHashes.add(actual); + } if (partitionHashes.size() == 1) { return partitionHashes.iterator().next(); } @@ -447,10 +463,6 @@ private String requiredFeatureSetHash(List partitionPlans) { Map featureDocuments = new java.util.TreeMap<>(); for (PartitionPlan partition : partitionPlans) { JsonNode features = partition.planRoot().path("requiredFeatures"); - if (!features.isArray()) { - throw new IllegalStateException( - "A partition with a distinct feature set must carry requiredFeatures"); - } features.forEach(feature -> { String key = requiredText(feature, "featureCode") + "\u0000" + requiredText(feature, "resolution"); diff --git a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssemblerTest.java b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssemblerTest.java index 4d7c97c5..81d43cd2 100644 --- a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssemblerTest.java +++ b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssemblerTest.java @@ -30,6 +30,11 @@ class StrategyBotCompiledPlanAssemblerTest { private static final Instant RELEASED_AT = Instant.parse("2026-08-04T13:30:00Z"); private static final String HASH_A = "a".repeat(64); private static final String HASH_B = "b".repeat(64); + private static final String EMPTY_FEATURE_SET_HASH = + "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"; + private static final String RSI_FEATURE_SET_HASH = + "160d96f04548e15fed4c1e23abb3ccbdd1e6f067ba94b4451a32ce8ca2a2e94f"; + private static final String FORGED_FEATURE_SET_HASH = "f".repeat(64); private final ObjectMapper objectMapper = new ObjectMapper(); private final StrategyBotCompiledPlanAssembler assembler = new StrategyBotCompiledPlanAssembler(); @@ -58,6 +63,104 @@ void publishesEveryCompiledPartitionUnderItsExactIdentity() { assertThat(sideOf(partitions.get(1).path("flows").get(0))).isEqualTo("SELL"); } + @Test + void preservesAValidSinglePartitionCompilerFeatureSetIdentity() { + ContractPlan plan = assemble(planWith(buyFlow("buy", List.of(AAPL)))); + + assertThat(parse(plan.planDocument()).path("requiredFeatureSetHash").asText()) + .isEqualTo("sha256:" + RSI_FEATURE_SET_HASH); + } + + @Test + void refusesASinglePartitionWhoseClaimedHashDoesNotMatchItsFeatureDefinitions() { + var root = (com.fasterxml.jackson.databind.node.ObjectNode) + planWith(buyFlow("buy", List.of(AAPL))); + root.put("requiredFeatureSetHash", FORGED_FEATURE_SET_HASH); + + assertThatThrownBy(() -> assemble(root)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("requiredFeatureSetHash does not match requiredFeatures"); + } + + @Test + void refusesWhenOneOfSeveralPartitionsChangesItsFeatureDefinitionWithoutRehashing() { + UUID secondPartition = UUID.fromString("41000000-0000-4000-8000-000000000002"); + JsonNode firstRoot = planWith(buyFlow("buy", List.of(AAPL))); + var secondRoot = (com.fasterxml.jackson.databind.node.ObjectNode) + planWith(sellFlow("sell", List.of(MSFT))); + ((com.fasterxml.jackson.databind.node.ObjectNode) + secondRoot.path("requiredFeatures").get(0)) + .put("definitionHash", "c".repeat(64)); + + assertThatThrownBy(() -> assembler.assemble( + List.of( + new StrategyBotCompiledPlanAssembler.PartitionPlan( + firstRoot, PARTITION_ID, 10_000, flowRecords(firstRoot)), + new StrategyBotCompiledPlanAssembler.PartitionPlan( + secondRoot, secondPartition, 10_000, flowRecords(secondRoot))), + catalog(feature("RSI_14", "rsi:1.0.0", "30m", 15)), + new BigDecimal("100000.00"), HASH_A, HASH_B, + "basic-launch-snapshot.v1", RELEASED_AT)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining(secondPartition.toString()) + .hasMessageContaining("requiredFeatureSetHash does not match requiredFeatures"); + } + + @Test + void refusesMultiplePartitionsThatShareOneForgedFeatureSetHash() { + UUID secondPartition = UUID.fromString("41000000-0000-4000-8000-000000000002"); + var firstRoot = (com.fasterxml.jackson.databind.node.ObjectNode) + planWith(buyFlow("buy", List.of(AAPL))); + var secondRoot = (com.fasterxml.jackson.databind.node.ObjectNode) + planWith(sellFlow("sell", List.of(MSFT))); + firstRoot.put("requiredFeatureSetHash", FORGED_FEATURE_SET_HASH); + secondRoot.put("requiredFeatureSetHash", FORGED_FEATURE_SET_HASH); + + assertThatThrownBy(() -> assembler.assemble( + List.of( + new StrategyBotCompiledPlanAssembler.PartitionPlan( + firstRoot, PARTITION_ID, 10_000, flowRecords(firstRoot)), + new StrategyBotCompiledPlanAssembler.PartitionPlan( + secondRoot, secondPartition, 10_000, flowRecords(secondRoot))), + catalog(feature("RSI_14", "rsi:1.0.0", "30m", 15)), + new BigDecimal("100000.00"), HASH_A, HASH_B, + "basic-launch-snapshot.v1", RELEASED_AT)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("requiredFeatureSetHash does not match requiredFeatures"); + } + + @Test + void refusesAPartitionThatOmitsItsCanonicalFeatureDefinitions() { + var root = (com.fasterxml.jackson.databind.node.ObjectNode) + planWith(buyFlow("buy", List.of(AAPL))); + root.remove("requiredFeatures"); + + assertThatThrownBy(() -> assemble(root)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("requiredFeatures must be an array"); + } + + @Test + void refusesPartitionsCompiledByDifferentCompilerVersions() { + UUID secondPartition = UUID.fromString("41000000-0000-4000-8000-000000000002"); + JsonNode firstRoot = planWith(buyFlow("buy", List.of(AAPL))); + var secondRoot = (com.fasterxml.jackson.databind.node.ObjectNode) + planWith(sellFlow("sell", List.of(MSFT))); + secondRoot.put("compilerVersion", "basic-compiler:2.0.0"); + + assertThatThrownBy(() -> assembler.assemble( + List.of( + new StrategyBotCompiledPlanAssembler.PartitionPlan( + firstRoot, PARTITION_ID, 10_000, flowRecords(firstRoot)), + new StrategyBotCompiledPlanAssembler.PartitionPlan( + secondRoot, secondPartition, 10_000, flowRecords(secondRoot))), + catalog(feature("RSI_14", "rsi:1.0.0", "30m", 15)), + new BigDecimal("100000.00"), HASH_A, HASH_B, + "basic-launch-snapshot.v1", RELEASED_AT)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("all partitions must use one compiler version"); + } + @Test void hashesTheUnionWhenPartitionsRequireDifferentFeatureSets() { UUID secondPartition = UUID.fromString("41000000-0000-4000-8000-000000000002"); @@ -70,13 +173,11 @@ void hashesTheUnionWhenPartitionsRequireDifferentFeatureSets() { .add(step(1, "TEST_CONDITION", "{}")) .add(step(2, "BASIC_EQUAL_ALLOCATION_ORDER", "{}"))); var directRoot = (com.fasterxml.jackson.databind.node.ObjectNode) planWith(directFlow); - directRoot.put("requiredFeatureSetHash", - "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"); + directRoot.put("requiredFeatureSetHash", EMPTY_FEATURE_SET_HASH); directRoot.set("requiredFeatures", objectMapper.createArrayNode()); var featureRoot = (com.fasterxml.jackson.databind.node.ObjectNode) planWith(sellFlow("feature", List.of(MSFT))); - featureRoot.put("requiredFeatureSetHash", - "160d96f04548e15fed4c1e23abb3ccbdd1e6f067ba94b4451a32ce8ca2a2e94f"); + featureRoot.put("requiredFeatureSetHash", RSI_FEATURE_SET_HASH); featureRoot.set("requiredFeatures", parse(""" [{"calculatorVersion":"rsi:1.0.0","definitionHash":"%s", "featureCode":"RSI_14","normalizedParameters":{"period":14}, @@ -394,7 +495,24 @@ private JsonNode planWith(JsonNode... flows) { var root = objectMapper.createObjectNode(); root.put("schemaVersion", "basic-compiled-plan.v1"); root.put("compilerVersion", "basic-compiler:1.0.0"); - root.put("requiredFeatureSetHash", HASH_A); + boolean requiresRsi = false; + for (JsonNode flow : flows) { + for (JsonNode step : flow.path("steps")) { + requiresRsi |= "BASIC_RSI_READ".equals(step.path("elementCode").asText()); + } + } + root.put( + "requiredFeatureSetHash", + requiresRsi ? RSI_FEATURE_SET_HASH : EMPTY_FEATURE_SET_HASH); + root.set( + "requiredFeatures", + requiresRsi + ? parse(""" + [{"calculatorVersion":"rsi:1.0.0","definitionHash":"%s", + "featureCode":"RSI_14","normalizedParameters":{"period":14}, + "outputValueType":"NUMBER","requiredHistoryPoints":15,"resolution":"30m"}] + """.formatted(HASH_B)) + : objectMapper.createArrayNode()); var array = root.putArray("flows"); for (JsonNode flow : flows) { array.add(flow); From 31c5f5578e6ea311da470cf1b881b510f815a7fa Mon Sep 17 00:00:00 2001 From: Ryan Min Date: Tue, 1 Sep 2026 13:02:52 +0900 Subject: [PATCH 05/13] test: prove partition feature union identities --- .../StrategyBotCompiledPlanAssemblerTest.java | 218 ++++++++++++++---- 1 file changed, 178 insertions(+), 40 deletions(-) diff --git a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssemblerTest.java b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssemblerTest.java index 81d43cd2..ebc7d811 100644 --- a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssemblerTest.java +++ b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/StrategyBotCompiledPlanAssemblerTest.java @@ -23,8 +23,14 @@ */ class StrategyBotCompiledPlanAssemblerTest { private static final UUID CATALOG_ID = UUID.fromString("40000000-0000-4000-8000-000000000001"); + private static final UUID OFFICIAL_CATALOG_ID = + UUID.fromString("0f5a0000-0000-4000-8000-000000000001"); private static final UUID PARTITION_ID = UUID.fromString("41000000-0000-4000-8000-000000000001"); private static final UUID FEATURE_ID = UUID.fromString("70000000-0000-4000-8000-000000000001"); + private static final UUID OFFICIAL_RSI_30M_FEATURE_ID = + UUID.fromString("ec37984b-6605-5560-8ea0-774c5b8e9626"); + private static final UUID OFFICIAL_RSI_1H_FEATURE_ID = + UUID.fromString("85f4f80f-be4e-d9dc-bd52-d4781ba5f30f"); private static final UUID AAPL = UUID.fromString("60000000-0000-4000-8000-000000000001"); private static final UUID MSFT = UUID.fromString("60000000-0000-4000-8000-000000000002"); private static final Instant RELEASED_AT = Instant.parse("2026-08-04T13:30:00Z"); @@ -34,6 +40,41 @@ class StrategyBotCompiledPlanAssemblerTest { "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945"; private static final String RSI_FEATURE_SET_HASH = "160d96f04548e15fed4c1e23abb3ccbdd1e6f067ba94b4451a32ce8ca2a2e94f"; + private static final String OFFICIAL_RSI_30M_FEATURE_DOCUMENT = """ + [{"calculatorVersion":"rsi:1.0.0", + "definitionHash":"sha256:250df12e46d233e7b8ece86c64df7a3941f0d70436aebe522b1387f15fb346dc", + "featureCode":"RSI_14", + "normalizedParameters":{"calendar_id":"XNYS", + "input_adjustment":"SPLIT_DIVIDEND_ADJUSTED", + "method":"SIMPLE_AVERAGE_BOUNDED_WINDOW","period":14,"price_field":"close"}, + "outputValueType":"NUMBER","requiredHistoryPoints":15,"resolution":"30m"}] + """; + private static final String OFFICIAL_RSI_1H_FEATURE_DOCUMENT = """ + [{"calculatorVersion":"rsi:1.0.0", + "definitionHash":"sha256:7e8c5600ff2bf07a043f797a50d6467f86fbdb56ee532c87929df97f246af2de", + "featureCode":"RSI_14", + "normalizedParameters":{"calendar_id":"XNYS", + "input_adjustment":"SPLIT_DIVIDEND_ADJUSTED", + "method":"SIMPLE_AVERAGE_BOUNDED_WINDOW","period":14,"price_field":"close"}, + "outputValueType":"NUMBER","requiredHistoryPoints":15,"resolution":"1h"}] + """; + private static final String RETIRED_OFFICIAL_RSI_30M_FEATURE_DOCUMENT = """ + [{"calculatorVersion":"rsi:1.0.0", + "definitionHash":"363f534dc77c6af0ebfe58f35be4fd2aa208906b1eaa36b550b17e9acb8692e4", + "featureCode":"RSI_14", + "normalizedParameters":{"calendar_id":"XNYS", + "input_adjustment":"SPLIT_DIVIDEND_ADJUSTED", + "method":"SIMPLE_AVERAGE_BOUNDED_WINDOW","period":14,"price_field":"close"}, + "outputValueType":"NUMBER","requiredHistoryPoints":15,"resolution":"30m"}] + """; + private static final String OFFICIAL_RSI_30M_FEATURE_SET_HASH = + "7b17d553083fd23a2dd846bc85fc9808ab207c303f6e94c809e63aa094159c0d"; + private static final String OFFICIAL_RSI_1H_FEATURE_SET_HASH = + "ac3758cba330676bea665e3b5338af37a64169f621bb52aa7c314526c930e1ca"; + private static final String RETIRED_OFFICIAL_RSI_30M_FEATURE_SET_HASH = + "403ce6386874ff9be98ed0e254bb750e1748ba1c6951498241609309829ae768"; + private static final String OFFICIAL_RSI_UNION_FEATURE_SET_HASH = + "1fe22f5c829fcfc1c341e7f3870dbe68eb9d2cb487d63abfd3a3e4b1cc2f8688"; private static final String FORGED_FEATURE_SET_HASH = "f".repeat(64); private final ObjectMapper objectMapper = new ObjectMapper(); @@ -162,53 +203,76 @@ void refusesPartitionsCompiledByDifferentCompilerVersions() { } @Test - void hashesTheUnionWhenPartitionsRequireDifferentFeatureSets() { + void hashesTheSortedUnionOfTwoDisjointNonEmptyCompilerFeatureDocuments() { UUID secondPartition = UUID.fromString("41000000-0000-4000-8000-000000000002"); - JsonNode directFlow = objectMapper.createObjectNode() - .put("key", "direct") - .put("container", "BUY") - .set( - "instrumentIds", objectMapper.createArrayNode().add(AAPL.toString())) - .set("steps", objectMapper.createArrayNode() - .add(step(1, "TEST_CONDITION", "{}")) - .add(step(2, "BASIC_EQUAL_ALLOCATION_ORDER", "{}"))); - var directRoot = (com.fasterxml.jackson.databind.node.ObjectNode) planWith(directFlow); - directRoot.put("requiredFeatureSetHash", EMPTY_FEATURE_SET_HASH); - directRoot.set("requiredFeatures", objectMapper.createArrayNode()); - var featureRoot = (com.fasterxml.jackson.databind.node.ObjectNode) - planWith(sellFlow("feature", List.of(MSFT))); - featureRoot.put("requiredFeatureSetHash", RSI_FEATURE_SET_HASH); - featureRoot.set("requiredFeatures", parse(""" - [{"calculatorVersion":"rsi:1.0.0","definitionHash":"%s", - "featureCode":"RSI_14","normalizedParameters":{"period":14}, - "outputValueType":"NUMBER","requiredHistoryPoints":15,"resolution":"30m"}] - """.formatted(HASH_B))); - BasicStrategyCatalog selected = new BasicStrategyCatalog( - catalog().version(), - List.of( - element("TEST_CONDITION", "TEST", "{}", "[]"), - element("BASIC_RSI_READ", "LOAD_FEATURE", - "{\"feature\":\"RSI_14\",\"resolution\":\"$resolution\"}", - "[\"RSI_14\"]"), - element("BASIC_VALUE_COMPARE", "COMPARE", - "{\"operator\":\"$operator\",\"threshold\":\"$threshold\"}", "[]"), - element("BASIC_EQUAL_ALLOCATION_ORDER", "EMIT_ORDER_CANDIDATE", - "{\"allocation\":\"EQUAL\",\"orderType\":\"MARKET\"," - + "\"side\":\"$container\"}", "[]")), - List.of(feature("RSI_14", "rsi:1.0.0", "30m", 15)), - catalog().instruments()); + var thirtyMinuteRoot = (com.fasterxml.jackson.databind.node.ObjectNode) + planWith(officialRsiFlow("thirty-minute", "BUY", List.of(AAPL), "30m")); + thirtyMinuteRoot.put("requiredFeatureSetHash", OFFICIAL_RSI_30M_FEATURE_SET_HASH); + thirtyMinuteRoot.set("requiredFeatures", parse(OFFICIAL_RSI_30M_FEATURE_DOCUMENT)); + var oneHourRoot = (com.fasterxml.jackson.databind.node.ObjectNode) + planWith(officialRsiFlow("one-hour", "SELL", List.of(MSFT), "1h")); + oneHourRoot.put("requiredFeatureSetHash", OFFICIAL_RSI_1H_FEATURE_SET_HASH); + oneHourRoot.set("requiredFeatures", parse(OFFICIAL_RSI_1H_FEATURE_DOCUMENT)); + BasicStrategyCatalog selected = officialRsiCatalog( + officialRsiFeature("30m"), officialRsiFeature("1h")); ContractPlan plan = assembler.assemble( List.of( new StrategyBotCompiledPlanAssembler.PartitionPlan( - directRoot, PARTITION_ID, 10_000, flowRecords(directRoot)), + thirtyMinuteRoot, PARTITION_ID, 10_000, + flowRecords(thirtyMinuteRoot, OFFICIAL_CATALOG_ID)), new StrategyBotCompiledPlanAssembler.PartitionPlan( - featureRoot, secondPartition, 10_000, flowRecords(featureRoot))), + oneHourRoot, secondPartition, 10_000, + flowRecords(oneHourRoot, OFFICIAL_CATALOG_ID))), selected, new BigDecimal("100000.00"), HASH_A, HASH_B, "basic-launch-snapshot.v1", RELEASED_AT); - assertThat(parse(plan.planDocument()).path("requiredFeatureSetHash").asText()) - .isEqualTo("sha256:160d96f04548e15fed4c1e23abb3ccbdd1e6f067ba94b4451a32ce8ca2a2e94f"); + JsonNode assembled = parse(plan.planDocument()); + assertThat(assembled.path("requiredFeatures")).isEqualTo(parse(""" + [ + {"requirementId":"rsi-14-pt1h", + "featureId":"85f4f80f-be4e-d9dc-bd52-d4781ba5f30f", + "featureVersion":"1.0.0", + "instruments":["60000000-0000-4000-8000-000000000002"], + "resolution":"PT1H","requiredObservations":14}, + {"requirementId":"rsi-14-pt30m", + "featureId":"ec37984b-6605-5560-8ea0-774c5b8e9626", + "featureVersion":"1.0.0", + "instruments":["60000000-0000-4000-8000-000000000001"], + "resolution":"PT30M","requiredObservations":14} + ] + """)); + assertThat(assembled.path("requiredFeatureSetHash").asText()) + .isEqualTo("sha256:" + OFFICIAL_RSI_UNION_FEATURE_SET_HASH) + .isNotEqualTo("sha256:" + OFFICIAL_RSI_30M_FEATURE_SET_HASH) + .isNotEqualTo("sha256:" + OFFICIAL_RSI_1H_FEATURE_SET_HASH); + } + + @Test + void refusesConflictingCanonicalDefinitionsForTheSameFeatureIdentityAndResolution() { + UUID secondPartition = UUID.fromString("41000000-0000-4000-8000-000000000002"); + var activeRoot = (com.fasterxml.jackson.databind.node.ObjectNode) + planWith(officialRsiFlow("active", "BUY", List.of(AAPL), "30m")); + activeRoot.put("requiredFeatureSetHash", OFFICIAL_RSI_30M_FEATURE_SET_HASH); + activeRoot.set("requiredFeatures", parse(OFFICIAL_RSI_30M_FEATURE_DOCUMENT)); + var retiredRoot = (com.fasterxml.jackson.databind.node.ObjectNode) + planWith(officialRsiFlow("retired", "SELL", List.of(MSFT), "30m")); + retiredRoot.put("requiredFeatureSetHash", RETIRED_OFFICIAL_RSI_30M_FEATURE_SET_HASH); + retiredRoot.set("requiredFeatures", parse(RETIRED_OFFICIAL_RSI_30M_FEATURE_DOCUMENT)); + + assertThatThrownBy(() -> assembler.assemble( + List.of( + new StrategyBotCompiledPlanAssembler.PartitionPlan( + activeRoot, PARTITION_ID, 10_000, + flowRecords(activeRoot, OFFICIAL_CATALOG_ID)), + new StrategyBotCompiledPlanAssembler.PartitionPlan( + retiredRoot, secondPartition, 10_000, + flowRecords(retiredRoot, OFFICIAL_CATALOG_ID))), + officialRsiCatalog(officialRsiFeature("30m")), + new BigDecimal("100000.00"), HASH_A, HASH_B, + "basic-launch-snapshot.v1", RELEASED_AT)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Partitions disagree about required feature RSI_14@30m"); } @Test @@ -477,6 +541,10 @@ private ContractPlan assembleWithCatalog(JsonNode planRoot, BasicStrategyCatalog } private List flowRecords(JsonNode planRoot) { + return flowRecords(planRoot, CATALOG_ID); + } + + private List flowRecords(JsonNode planRoot, UUID catalogId) { List flows = new java.util.ArrayList<>(); int order = 0; for (JsonNode flowNode : planRoot.path("flows")) { @@ -485,7 +553,7 @@ private List flowRecords(JsonNode planRoot) { flows.add(new Flow( UUID.nameUUIDFromBytes(flowNode.path("key").asText().getBytes( java.nio.charset.StandardCharsets.UTF_8)), - flowNode.path("key").asText(), CATALOG_ID, UUID.randomUUID(), "{}", "{}", + flowNode.path("key").asText(), catalogId, UUID.randomUUID(), "{}", "{}", HASH_A, HASH_B, HASH_A, instruments, List.of(), order++)); } return List.copyOf(flows); @@ -541,6 +609,24 @@ private JsonNode flow(String key, String container, List instruments) { return node; } + private JsonNode officialRsiFlow( + String key, String container, List instruments, String resolution) { + var node = objectMapper.createObjectNode(); + node.put("key", key); + node.put("container", container); + var ids = node.putArray("instrumentIds"); + instruments.stream().map(UUID::toString).sorted().forEach(ids::add); + node.set("steps", objectMapper.createArrayNode() + .add(step(1, "BASIC_RSI_CROSS", """ + {"resolution":"%s","direction":"UP","period":"14","threshold":"50"} + """.formatted(resolution))) + .add(step(2, "BASIC_EQUAL_ALLOCATION_ORDER", """ + {"orderPercent":"25","maxPositionPercent":"40","executionMode":"1회만", + "waitMode":"조건 재충족","waitInterval":"1","maxExecutions":"1"} + """))); + return node; + } + private JsonNode step(int sequence, String elementCode, String parameters) { var node = objectMapper.createObjectNode(); node.put("sequence", sequence); @@ -568,6 +654,53 @@ private BasicStrategyCatalog catalog(StrategyFeatureDefinition... features) { new SupportedInstrument(MSFT, "STOCK", "XNAS", "USD", "MSFT"))); } + private BasicStrategyCatalog officialRsiCatalog(StrategyFeatureDefinition... features) { + return new BasicStrategyCatalog( + new ElementCatalogVersion( + OFFICIAL_CATALOG_ID, "basic/v1", "basic-semantic/v1", + "basic-elements:2026-08-25", "alpaca-sip/v1", + "sha256:6f564e46b2696158c4c9ae2866a7fe6e02f4cb06849d7612a4032c632227d320", + Instant.parse("2026-08-25T00:00:00Z"), null), + List.of( + element(OFFICIAL_CATALOG_ID, "BASIC_RSI_CROSS", "RSI_CROSS", + "{\"direction\":\"$direction\",\"period\":\"$period\"," + + "\"resolution\":\"$resolution\",\"threshold\":\"$threshold\"}", + "[\"RSI_14\"]"), + element(OFFICIAL_CATALOG_ID, "BASIC_EQUAL_ALLOCATION_ORDER", + "EMIT_ORDER_CANDIDATE", + "{\"allocation\":\"EQUAL\",\"executionMode\":\"$executionMode\"," + + "\"maxExecutions\":\"$maxExecutions\"," + + "\"maxPositionPercent\":\"$maxPositionPercent\"," + + "\"orderPercent\":\"$orderPercent\",\"orderType\":\"MARKET\"," + + "\"side\":\"$container\",\"timeInForce\":\"DAY\"," + + "\"waitInterval\":\"$waitInterval\",\"waitMode\":\"$waitMode\"}", + "[]")), + List.of(features), + List.of( + new SupportedInstrument(AAPL, "STOCK", "XNAS", "USD", "AAPL"), + new SupportedInstrument(MSFT, "STOCK", "XNAS", "USD", "MSFT"))); + } + + private static StrategyFeatureDefinition officialRsiFeature(String resolution) { + return switch (resolution) { + case "30m" -> new StrategyFeatureDefinition( + OFFICIAL_RSI_30M_FEATURE_ID, OFFICIAL_CATALOG_ID, "RSI_14", "rsi:1.0.0", "30m", + "{\"calendar_id\":\"XNYS\",\"input_adjustment\":\"SPLIT_DIVIDEND_ADJUSTED\"," + + "\"method\":\"SIMPLE_AVERAGE_BOUNDED_WINDOW\",\"period\":14," + + "\"price_field\":\"close\"}", + "NUMBER", 15, + "sha256:250df12e46d233e7b8ece86c64df7a3941f0d70436aebe522b1387f15fb346dc"); + case "1h" -> new StrategyFeatureDefinition( + OFFICIAL_RSI_1H_FEATURE_ID, OFFICIAL_CATALOG_ID, "RSI_14", "rsi:1.0.0", "1h", + "{\"calendar_id\":\"XNYS\",\"input_adjustment\":\"SPLIT_DIVIDEND_ADJUSTED\"," + + "\"method\":\"SIMPLE_AVERAGE_BOUNDED_WINDOW\",\"period\":14," + + "\"price_field\":\"close\"}", + "NUMBER", 15, + "sha256:7e8c5600ff2bf07a043f797a50d6467f86fbdb56ee532c87929df97f246af2de"); + default -> throw new IllegalArgumentException("unsupported official RSI resolution: " + resolution); + }; + } + private static StrategyFeatureDefinition feature( String code, String calculatorVersion, String resolution, int historyPoints) { return new StrategyFeatureDefinition( @@ -578,8 +711,13 @@ private static StrategyFeatureDefinition feature( private static StrategyElementDefinition element( String code, String operation, String arguments, String features) { + return element(CATALOG_ID, code, operation, arguments, features); + } + + private static StrategyElementDefinition element( + UUID catalogId, String code, String operation, String arguments, String features) { return new StrategyElementDefinition( - UUID.nameUUIDFromBytes(code.getBytes(java.nio.charset.StandardCharsets.UTF_8)), CATALOG_ID, + UUID.nameUUIDFromBytes(code.getBytes(java.nio.charset.StandardCharsets.UTF_8)), catalogId, code, "BLOCK", "{}", "{}", "{}", "{\"containers\":[\"BUY\",\"SELL\"],\"runtime\":{\"operation\":\"" + operation + "\"," + "\"arguments\":" + arguments + "}," From 466b9d8523089d2acab5ad60a4f3b68707060149 Mon Sep 17 00:00:00 2001 From: Ryan Min Date: Tue, 1 Sep 2026 13:45:35 +0900 Subject: [PATCH 06/13] fix: keep backtest inputs instrument scoped --- .../OfficialBacktestInputSelector.java | 5 ++- .../StrategyBotIndependentE2ETest.java | 2 +- ...ableStrategyReleaseCommandServiceTest.java | 2 +- .../OfficialBacktestInputSelectorTest.java | 39 +++++++++++++++++-- 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelector.java b/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelector.java index 1fa1fb5f..7054d8d1 100644 --- a/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelector.java +++ b/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelector.java @@ -95,8 +95,9 @@ private static Selection selectDatasets( coverageEnd, candidates.stream() .filter(dataset -> resolution.equals(normalizeResolution(dataset.resolution()))) - .filter(dataset -> dataset.instrumentId() == null - || dataset.instrumentId().equals(requiredInstrument)) + .filter(dataset -> requiredInstrument == null + ? dataset.instrumentId() == null + : requiredInstrument.equals(dataset.instrumentId())) .toList())); } } diff --git a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/journey/StrategyBotIndependentE2ETest.java b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/journey/StrategyBotIndependentE2ETest.java index c6a36ef0..4c558435 100644 --- a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/journey/StrategyBotIndependentE2ETest.java +++ b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/journey/StrategyBotIndependentE2ETest.java @@ -124,7 +124,7 @@ void createsValidatesReleasesRunsAndPermanentlyStopsABasicStrategyBot() { java.time.LocalDate.parse("2025-01-01"), java.time.LocalDate.parse("2025-12-31"), "market-bars/1", NOW.minusSeconds(60))), List.of(new StrategyReleaseInputCatalog.Dataset( - DATASET_ID, "ALPACA_SIP_ALL_30M", "ADJUSTED", "30m", 1, + DATASET_ID, INSTRUMENT_ID, "ALPACA_SIP_ALL_30M", "ADJUSTED", "30m", 1, java.time.LocalDate.parse("2025-01-01"), java.time.LocalDate.parse("2025-12-31"), "market-bars/1", NOW.minusSeconds(30))), observedAt), diff --git a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/ImmutableStrategyReleaseCommandServiceTest.java b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/ImmutableStrategyReleaseCommandServiceTest.java index e686d24e..c63e3f70 100644 --- a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/ImmutableStrategyReleaseCommandServiceTest.java +++ b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/ImmutableStrategyReleaseCommandServiceTest.java @@ -239,7 +239,7 @@ private static StrategyReleaseInputCatalogQueryPort inputCatalogPort() { java.time.LocalDate.parse("2025-01-01"), java.time.LocalDate.parse("2025-12-31"), "market-bars/1", NOW.minusSeconds(60))), List.of(new StrategyReleaseInputCatalog.Dataset( - DATASET_ID, "ALPACA_SIP_ALL_30M", "ADJUSTED", "30m", 1, + DATASET_ID, AAPL_ID, "ALPACA_SIP_ALL_30M", "ADJUSTED", "30m", 1, java.time.LocalDate.parse("2025-01-01"), java.time.LocalDate.parse("2025-12-31"), "market-bars/1", NOW.minusSeconds(30))), observedAt); diff --git a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelectorTest.java b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelectorTest.java index a47ab09f..841db50c 100644 --- a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelectorTest.java +++ b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelectorTest.java @@ -182,6 +182,33 @@ void neverUsesAnInstrumentScopedManifestForAnotherInstrument() { assertThat(selected.datasets()).extracting(Dataset::id).containsExactly(aaplBars); } + @Test + void exactInstrumentRequirementNeverSelectsANewerUniverseWideCrossProduct() { + UUID aapl = UUID.fromString("73000000-0000-4000-8000-000000000001"); + UUID universeBars = UUID.fromString("73000000-0000-4000-8000-000000000002"); + UUID aaplBars = UUID.fromString("73000000-0000-4000-8000-000000000003"); + var catalog = new StrategyReleaseInputCatalog( + List.of(policy("official-v1", NOW.minusSeconds(60))), + List.of( + dataset(universeBars, "ADJUSTED", "30m", 9, + "2024-01-01", "2024-02-01", NOW.minusSeconds(10)), + new Dataset(aaplBars, aapl, "AAPL", "ADJUSTED", "30m", 1, + LocalDate.parse("2024-01-01"), LocalDate.parse("2024-02-01"), + "market-bars/1", NOW.minusSeconds(20))), + NOW); + String plan = """ + {"executionSnapshot":{"partitions":[{"flows":[{ + "officialInstrumentIds":["%s"], + "steps":[{"arguments":{"resolution":"30m"}}] + }]}]}} + """.formatted(aapl); + + var selected = OfficialBacktestInputSelector.select( + plan, LocalDate.parse("2024-01-05"), LocalDate.parse("2024-01-25"), catalog); + + assertThat(selected.datasets()).extracting(Dataset::id).containsExactly(aaplBars); + } + @Test void selectsOnlyTheManifestsRequiredByIndependentThirtyMinuteFourHourAndDailyFlows() { UUID aapl = UUID.fromString("72000000-0000-4000-8000-000000000001"); @@ -194,10 +221,16 @@ void selectsOnlyTheManifestsRequiredByIndependentThirtyMinuteFourHourAndDailyFlo var catalog = new StrategyReleaseInputCatalog( List.of(policy("official-v1", NOW.minusSeconds(60))), List.of( - dataset(bars30m, "ADJUSTED", "30m", NOW.minusSeconds(20)), + new Dataset(bars30m, aapl, "AAPL", "ADJUSTED", "30m", 1, + LocalDate.parse("2024-01-01"), LocalDate.parse("2024-02-01"), + "market-bars/1", NOW.minusSeconds(20)), dataset(unused1h, "ADJUSTED", "1h", NOW.minusSeconds(20)), - dataset(bars4h, "ADJUSTED", "4h", NOW.minusSeconds(20)), - dataset(bars1d, "ADJUSTED", "1d", NOW.minusSeconds(20))), + new Dataset(bars4h, msft, "MSFT", "ADJUSTED", "4h", 1, + LocalDate.parse("2024-01-01"), LocalDate.parse("2024-02-01"), + "market-bars/1", NOW.minusSeconds(20)), + new Dataset(bars1d, meta, "META", "ADJUSTED", "1d", 1, + LocalDate.parse("2024-01-01"), LocalDate.parse("2024-02-01"), + "market-bars/1", NOW.minusSeconds(20))), NOW); String plan = """ {"executionSnapshot":{"partitions":[{"flows":[ From 468bb969acb05e9d7d522bc6927637eb69125125 Mon Sep 17 00:00:00 2001 From: Ryan Min Date: Tue, 1 Sep 2026 15:54:26 +0900 Subject: [PATCH 07/13] test: align exact release input fixtures --- .../v1/AssembledCompiledPlanContractTest.java | 8 +++++- ...tomBacktestJooqAdapterIntegrationTest.java | 26 +++++++++---------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/modules/backend-messaging/src/test/java/com/idea2strategy/backend/messaging/strategybot/v1/AssembledCompiledPlanContractTest.java b/modules/backend-messaging/src/test/java/com/idea2strategy/backend/messaging/strategybot/v1/AssembledCompiledPlanContractTest.java index fe6f07c8..937f3d5d 100644 --- a/modules/backend-messaging/src/test/java/com/idea2strategy/backend/messaging/strategybot/v1/AssembledCompiledPlanContractTest.java +++ b/modules/backend-messaging/src/test/java/com/idea2strategy/backend/messaging/strategybot/v1/AssembledCompiledPlanContractTest.java @@ -36,6 +36,8 @@ class AssembledCompiledPlanContractTest { private static final UUID AAPL = UUID.fromString("60000000-0000-4000-8000-000000000001"); private static final String HASH_A = "a".repeat(64); private static final String HASH_B = "b".repeat(64); + private static final String RSI_FEATURE_SET_HASH = + "160d96f04548e15fed4c1e23abb3ccbdd1e6f067ba94b4451a32ce8ca2a2e94f"; private static final Instant RELEASED_AT = Instant.parse("2026-08-04T13:30:00Z"); @Test @@ -185,7 +187,11 @@ private static String assembledWith(BigDecimal initialCashAmount, Instant releas private static String compiledPlan() { return "{\"schemaVersion\":\"basic-compiled-plan.v1\",\"compilerVersion\":\"basic-compiler:1.0.0\"," - + "\"requiredFeatureSetHash\":\"" + HASH_A + "\",\"flows\":[{" + + "\"requiredFeatureSetHash\":\"" + RSI_FEATURE_SET_HASH + "\"," + + "\"requiredFeatures\":[{\"calculatorVersion\":\"rsi:1.0.0\"," + + "\"definitionHash\":\"" + HASH_B + "\",\"featureCode\":\"RSI_14\"," + + "\"normalizedParameters\":{\"period\":14},\"outputValueType\":\"NUMBER\"," + + "\"requiredHistoryPoints\":15,\"resolution\":\"30m\"}],\"flows\":[{" + "\"key\":\"buy\",\"container\":\"BUY\",\"instrumentIds\":[\"" + AAPL + "\"],\"steps\":[" + "{\"sequence\":1,\"elementCode\":\"BASIC_RSI_READ\",\"parameters\":{\"resolution\":\"30m\"}}," + "{\"sequence\":2,\"elementCode\":\"BASIC_VALUE_COMPARE\"," diff --git a/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/backtest/CustomBacktestJooqAdapterIntegrationTest.java b/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/backtest/CustomBacktestJooqAdapterIntegrationTest.java index 41ccdb3f..0b3891fb 100644 --- a/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/backtest/CustomBacktestJooqAdapterIntegrationTest.java +++ b/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/backtest/CustomBacktestJooqAdapterIntegrationTest.java @@ -90,12 +90,12 @@ void seed() { jdbc.update("delete from market_data.dataset_manifests where id = ?", FEATURE_MANIFEST); jdbc.update("delete from market_data.pipeline_runs where id = ?", PIPELINE); jdbc.update("delete from market_data.feature_definitions where id = ?", FEATURE); - jdbc.update("delete from market_data.instruments where id = ?", INSTRUMENT); jdbc.update("delete from market_data.dataset_objects where id in (?, ?)", DATASET_OBJECT, OLDER_DATASET_OBJECT); jdbc.update("delete from storage.objects where id in (?, ?)", STORAGE_OBJECT, OLDER_STORAGE_OBJECT); jdbc.update("delete from market_data.dataset_manifests where id = ?", DATASET); jdbc.update("delete from market_data.dataset_manifests where id = ?", OLDER_DATASET); jdbc.update("delete from market_data.dataset_manifests where id = ?", EMPTY_NEWER_DATASET); + jdbc.update("delete from market_data.instruments where id = ?", INSTRUMENT); jdbc.update("delete from market_data.feeds where provider_id = ? and code = 'FEATURE_RSI_14_1D_RSI_1_0_0'", PROVIDER); jdbc.update("delete from market_data.feeds where id in (?, ?)", FEED, FEATURE_FEED); jdbc.update("delete from trading.fee_policy_versions where id = ?", FEE); @@ -121,27 +121,30 @@ void seed() { + "values (?, ?, 'FEATURE_RSI_14_1D_RSI_1_0_0', 'FEATURE_SERIES', '1d', 'UTC', " + "'rsi-1.0.0+feature-series.parquet.v1', ?)", FEATURE_FEED, PROVIDER, at); + jdbc.update("insert into market_data.instruments " + + "(id, asset_type, primary_exchange_mic, currency_code) values (?, 'STOCK', 'XNAS', 'USD')", + INSTRUMENT); jdbc.update( "insert into market_data.dataset_manifests " - + "(id, feed_id, data_layer, resolution, revision_number, status, period_start, period_end, " + + "(id, feed_id, instrument_id, data_layer, resolution, revision_number, status, period_start, period_end, " + "schema_version, dataset_hash, created_at, available_at) " - + "values (?, ?, 'ADJUSTED', '1d', 1, 'AVAILABLE', '2024-01-01T05:00:00Z', " + + "values (?, ?, ?, 'ADJUSTED', '1d', 1, 'AVAILABLE', '2024-01-01T05:00:00Z', " + "'2025-01-01T04:59:59Z', 'v1', ?, ?, ?)", - DATASET, FEED, "a".repeat(64), at, at); + DATASET, FEED, INSTRUMENT, "a".repeat(64), at, at); jdbc.update( "insert into market_data.dataset_manifests " - + "(id, feed_id, data_layer, resolution, revision_number, status, period_start, period_end, " + + "(id, feed_id, instrument_id, data_layer, resolution, revision_number, status, period_start, period_end, " + "schema_version, dataset_hash, created_at, available_at) " - + "values (?, ?, 'ADJUSTED', '1d', 2, 'AVAILABLE', '2024-01-01T05:00:00Z', " + + "values (?, ?, ?, 'ADJUSTED', '1d', 2, 'AVAILABLE', '2024-01-01T05:00:00Z', " + "'2025-01-01T04:59:59Z', 'v1', ?, ?, ?)", - OLDER_DATASET, FEED, "8".repeat(64), at.minusDays(2), at.minusDays(1)); + OLDER_DATASET, FEED, INSTRUMENT, "8".repeat(64), at.minusDays(2), at.minusDays(1)); jdbc.update( "insert into market_data.dataset_manifests " - + "(id, feed_id, data_layer, resolution, revision_number, status, period_start, period_end, " + + "(id, feed_id, instrument_id, data_layer, resolution, revision_number, status, period_start, period_end, " + "schema_version, dataset_hash, created_at, available_at) " - + "values (?, ?, 'ADJUSTED', '1d', 3, 'AVAILABLE', '2024-01-01T05:00:00Z', " + + "values (?, ?, ?, 'ADJUSTED', '1d', 3, 'AVAILABLE', '2024-01-01T05:00:00Z', " + "'2025-01-01T04:59:59Z', 'v1', ?, ?, ?)", - EMPTY_NEWER_DATASET, FEED, "7".repeat(64), at.minusHours(1), at.minusHours(1)); + EMPTY_NEWER_DATASET, FEED, INSTRUMENT, "7".repeat(64), at.minusHours(1), at.minusHours(1)); insertMarketObject(STORAGE_OBJECT, DATASET_OBJECT, DATASET, "market/main.parquet", "6".repeat(64), at); insertMarketObject(OLDER_STORAGE_OBJECT, OLDER_DATASET_OBJECT, OLDER_DATASET, "market/revision-2.parquet", "5".repeat(64), at.minusDays(1)); @@ -149,9 +152,6 @@ void seed() { + "(id, language_version, schema_version, catalog_version, data_requirement_version, " + "definition_hash, published_at) values (?, 'basic/v1', 'schema/v1', 'catalog/v1', " + "'data/v1', ?, ?)", CATALOG, "b".repeat(64), at.minusDays(1)); - jdbc.update("insert into market_data.instruments " - + "(id, asset_type, primary_exchange_mic, currency_code) values (?, 'STOCK', 'XNAS', 'USD')", - INSTRUMENT); jdbc.update("insert into market_data.feature_definitions " + "(id, element_catalog_version_id, feature_code, calculator_version, resolution, " + "normalized_parameters, output_value_type, required_history_points, definition_hash) " From adf470647319f2e3305c4595a366d4f1b0aa9d7f Mon Sep 17 00:00:00 2001 From: Ryan Min Date: Wed, 2 Sep 2026 14:21:10 +0900 Subject: [PATCH 08/13] fix(migrations): add narrow backtest cleanup capability --- .../migration/DatabaseAccessPolicy.java | 20 + ...ine_backtest_object_cleanup_capability.sql | 404 ++++++++++++++++++ ...nupCapabilityMigrationIntegrationTest.java | 187 ++++++++ ...CanonicalMigrationBundleAssemblerTest.java | 4 + .../migration/DatabaseAccessPolicyTest.java | 15 + .../migration/MigrationPolicyTest.java | 3 +- 6 files changed, 632 insertions(+), 1 deletion(-) create mode 100644 db-migration/src/main/resources/db/migration/V20260902000000__pipeline_backtest_object_cleanup_capability.sql create mode 100644 db-migration/src/test/java/com/idea2strategy/backend/migration/BacktestObjectCleanupCapabilityMigrationIntegrationTest.java diff --git a/db-migration/src/main/java/com/idea2strategy/backend/migration/DatabaseAccessPolicy.java b/db-migration/src/main/java/com/idea2strategy/backend/migration/DatabaseAccessPolicy.java index e14b4451..066c719f 100644 --- a/db-migration/src/main/java/com/idea2strategy/backend/migration/DatabaseAccessPolicy.java +++ b/db-migration/src/main/java/com/idea2strategy/backend/migration/DatabaseAccessPolicy.java @@ -10,6 +10,8 @@ public final class DatabaseAccessPolicy { public static final String RUNTIME_GRANTS_FILE = "R__database_runtime_grants.sql"; + public static final String BACKTEST_OBJECT_CLEANUP_FUNCTION = + "\"storage\".\"prepare_backtest_object_cleanup\"(jsonb)"; private static final String ROLE_PREFIX = "idea2strategy_"; private static final Pattern CREATE_TABLE = Pattern.compile( "(?i)CREATE\\s+TABLE\\s+(?:IF\\s+NOT\\s+EXISTS\\s+)?" @@ -231,6 +233,24 @@ public static String runtimeGrantSql(List migrationSql) { .append(" TO ").append(roleName).append(";\n"); } } + // storage.objects remains non-deletable by every application role. Backtest + // compensation is a transaction-scoped, namespace-fenced SECURITY DEFINER + // capability installed by a forward migration, so only EXECUTE is exposed. + sql.append("REVOKE ALL ON FUNCTION ") + .append(BACKTEST_OBJECT_CLEANUP_FUNCTION) + .append(" FROM PUBLIC;\n"); + for (var role : ApplicationRole.values()) { + sql.append("REVOKE ALL ON FUNCTION ") + .append(BACKTEST_OBJECT_CLEANUP_FUNCTION) + .append(" FROM ") + .append(databaseRole(role)) + .append(";\n"); + } + sql.append("GRANT EXECUTE ON FUNCTION ") + .append(BACKTEST_OBJECT_CLEANUP_FUNCTION) + .append(" TO ") + .append(databaseRole(ApplicationRole.BACKTEST)) + .append(";\n"); return sql.toString(); } diff --git a/db-migration/src/main/resources/db/migration/V20260902000000__pipeline_backtest_object_cleanup_capability.sql b/db-migration/src/main/resources/db/migration/V20260902000000__pipeline_backtest_object_cleanup_capability.sql new file mode 100644 index 00000000..4a21b02d --- /dev/null +++ b/db-migration/src/main/resources/db/migration/V20260902000000__pipeline_backtest_object_cleanup_capability.sql @@ -0,0 +1,404 @@ +-- The backtest worker must compensate only its own uncommitted object versions. +-- It deliberately has no DELETE on storage.objects and no SELECT on every table +-- that may acquire a future FK to storage.objects. This transaction-scoped +-- capability performs that narrow operation with migration-owner privileges. + +CREATE OR REPLACE FUNCTION storage.prepare_backtest_object_cleanup(p_candidates jsonb) +RETURNS SETOF storage.objects +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +SET lock_timeout = '5s' +AS $cleanup$ +DECLARE + v_after jsonb; + v_before jsonb; + v_candidate_count integer; + v_candidate_ids uuid[]; + v_deleted_count integer; + v_existing_count integer; + v_reference_exists boolean; + v_source record; + v_unique_count integer; +BEGIN + IF jsonb_typeof(p_candidates) IS DISTINCT FROM 'array' + OR jsonb_array_length(p_candidates) = 0 THEN + RAISE EXCEPTION 'backtest object cleanup candidates must be a non-empty JSON array'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM jsonb_array_elements(p_candidates) AS offered(candidate) + WHERE jsonb_typeof(offered.candidate) IS DISTINCT FROM 'object' + OR ( + SELECT array_agg(key ORDER BY key) + FROM jsonb_object_keys(offered.candidate) AS keys(key) + ) IS DISTINCT FROM ARRAY[ + 'bucket_name', + 'content_hash', + 'object_id', + 'object_key', + 'provider_version_id', + 'storage_provider' + ]::text[] + ) THEN + RAISE EXCEPTION 'backtest object cleanup candidate shape is invalid'; + END IF; + + SELECT count(*), count(DISTINCT candidate.object_id), + array_agg(candidate.object_id ORDER BY candidate.object_id) + INTO v_candidate_count, v_unique_count, v_candidate_ids + FROM jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text + ); + + IF v_candidate_count IS DISTINCT FROM v_unique_count THEN + RAISE EXCEPTION 'backtest object cleanup candidate ids must be unique'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text + ) + WHERE candidate.object_id IS NULL + OR nullif(candidate.storage_provider, '') IS NULL + OR nullif(candidate.bucket_name, '') IS NULL + OR nullif(candidate.object_key, '') IS NULL + OR nullif(candidate.provider_version_id, '') IS NULL + OR candidate.content_hash !~ '^[0-9a-f]{64}$' + OR NOT ( + ( + candidate.object_key ~ + '^backtest-results/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/[0-9a-f]{64}[.]json$' + AND candidate.object_key LIKE '%/' || candidate.content_hash || '.json' + ) + OR + ( + candidate.object_key ~ + '^backtest-results/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/[A-Z][A-Z0-9_]{0,49}/week_start=[0-9]{4}-[0-9]{2}-[0-9]{2}/part=[0-9]{4}/[0-9a-f]{64}[.]parquet$' + AND candidate.object_key LIKE '%/' || candidate.content_hash || '.parquet' + ) + ) + ) THEN + RAISE EXCEPTION + 'backtest object cleanup requires exact identities in the canonical backtest object namespace'; + END IF; + + WITH foreign_keys AS ( + SELECT fk.oid AS constraint_oid, + fk.conname AS constraint_name, + source.oid AS source_oid, + source_namespace.nspname AS source_schema, + source.relname AS source_table, + source.relkind AS source_relkind, + source.relispartition AS source_is_partition, + EXISTS ( + SELECT 1 + FROM pg_inherits AS inheritance + WHERE inheritance.inhrelid = source.oid + OR inheritance.inhparent = source.oid + ) AS source_has_inheritance, + ARRAY( + SELECT source_column.attname + FROM unnest(fk.conkey) WITH ORDINALITY AS source_key(attnum, ordinality) + JOIN pg_attribute AS source_column + ON source_column.attrelid = source.oid + AND source_column.attnum = source_key.attnum + ORDER BY source_key.ordinality + ) AS source_columns, + target.oid AS target_oid, + target.relkind AS target_relkind, + target.relispartition AS target_is_partition, + EXISTS ( + SELECT 1 + FROM pg_inherits AS inheritance + WHERE inheritance.inhrelid = target.oid + OR inheritance.inhparent = target.oid + ) AS target_has_inheritance, + ARRAY( + SELECT target_column.attname + FROM unnest(fk.confkey) WITH ORDINALITY AS target_key(attnum, ordinality) + JOIN pg_attribute AS target_column + ON target_column.attrelid = target.oid + AND target_column.attnum = target_key.attnum + ORDER BY target_key.ordinality + ) AS target_columns, + fk.convalidated AS validated, + fk.condeferrable AS deferrable, + fk.confdeltype AS delete_action + FROM pg_constraint AS fk + JOIN pg_class AS source ON source.oid = fk.conrelid + JOIN pg_namespace AS source_namespace + ON source_namespace.oid = source.relnamespace + JOIN pg_class AS target ON target.oid = fk.confrelid + WHERE fk.contype = 'f' + AND fk.confrelid = 'storage.objects'::regclass + ) + SELECT coalesce( + jsonb_agg(to_jsonb(foreign_key) ORDER BY + foreign_key.source_schema, + foreign_key.source_table, + foreign_key.constraint_name, + foreign_key.constraint_oid), + '[]'::jsonb + ) + INTO v_before + FROM foreign_keys AS foreign_key; + + FOR v_source IN + SELECT reference + FROM jsonb_array_elements(v_before) AS reference_rows(reference) + LOOP + IF jsonb_array_length(v_source.reference->'source_columns') <> 1 + OR v_source.reference->'target_columns' <> '["id"]'::jsonb + OR v_source.reference->>'source_relkind' <> 'r' + OR (v_source.reference->>'source_is_partition')::boolean + OR (v_source.reference->>'source_has_inheritance')::boolean + OR v_source.reference->>'target_relkind' <> 'r' + OR (v_source.reference->>'target_is_partition')::boolean + OR (v_source.reference->>'target_has_inheritance')::boolean + OR v_source.reference->>'delete_action' <> 'a' THEN + RAISE EXCEPTION + 'unsupported storage.objects foreign-key shape at %.% (constraint %); cleanup fails closed', + v_source.reference->>'source_schema', + v_source.reference->>'source_table', + v_source.reference->>'constraint_name'; + END IF; + END LOOP; + + -- Source relations come first. An ALTER TABLE that already owns a source + -- lock may then acquire the target and finish; cleanup never holds the target + -- while waiting for that source, so the inverse ordering cannot deadlock. + FOR v_source IN + SELECT DISTINCT reference->>'source_schema' AS source_schema, + reference->>'source_table' AS source_table, + (reference->>'source_oid')::oid AS source_oid + FROM jsonb_array_elements(v_before) AS reference_rows(reference) + ORDER BY source_schema, source_table, source_oid + LOOP + BEGIN + EXECUTE format( + 'LOCK TABLE %I.%I IN ACCESS SHARE MODE', + v_source.source_schema, + v_source.source_table + ); + EXCEPTION + WHEN undefined_table THEN + RAISE EXCEPTION + 'storage.objects foreign-key catalog changed while cleanup acquired source locks'; + END; + END LOOP; + + LOCK TABLE storage.objects IN SHARE UPDATE EXCLUSIVE MODE; + + WITH foreign_keys AS ( + SELECT fk.oid AS constraint_oid, + fk.conname AS constraint_name, + source.oid AS source_oid, + source_namespace.nspname AS source_schema, + source.relname AS source_table, + source.relkind AS source_relkind, + source.relispartition AS source_is_partition, + EXISTS ( + SELECT 1 + FROM pg_inherits AS inheritance + WHERE inheritance.inhrelid = source.oid + OR inheritance.inhparent = source.oid + ) AS source_has_inheritance, + ARRAY( + SELECT source_column.attname + FROM unnest(fk.conkey) WITH ORDINALITY AS source_key(attnum, ordinality) + JOIN pg_attribute AS source_column + ON source_column.attrelid = source.oid + AND source_column.attnum = source_key.attnum + ORDER BY source_key.ordinality + ) AS source_columns, + target.oid AS target_oid, + target.relkind AS target_relkind, + target.relispartition AS target_is_partition, + EXISTS ( + SELECT 1 + FROM pg_inherits AS inheritance + WHERE inheritance.inhrelid = target.oid + OR inheritance.inhparent = target.oid + ) AS target_has_inheritance, + ARRAY( + SELECT target_column.attname + FROM unnest(fk.confkey) WITH ORDINALITY AS target_key(attnum, ordinality) + JOIN pg_attribute AS target_column + ON target_column.attrelid = target.oid + AND target_column.attnum = target_key.attnum + ORDER BY target_key.ordinality + ) AS target_columns, + fk.convalidated AS validated, + fk.condeferrable AS deferrable, + fk.confdeltype AS delete_action + FROM pg_constraint AS fk + JOIN pg_class AS source ON source.oid = fk.conrelid + JOIN pg_namespace AS source_namespace + ON source_namespace.oid = source.relnamespace + JOIN pg_class AS target ON target.oid = fk.confrelid + WHERE fk.contype = 'f' + AND fk.confrelid = 'storage.objects'::regclass + ) + SELECT coalesce( + jsonb_agg(to_jsonb(foreign_key) ORDER BY + foreign_key.source_schema, + foreign_key.source_table, + foreign_key.constraint_name, + foreign_key.constraint_oid), + '[]'::jsonb + ) + INTO v_after + FROM foreign_keys AS foreign_key; + + IF v_before IS DISTINCT FROM v_after THEN + RAISE EXCEPTION + 'storage.objects foreign-key catalog changed while cleanup acquired locks'; + END IF; + + FOR v_source IN + SELECT reference + FROM jsonb_array_elements(v_after) AS reference_rows(reference) + LOOP + IF jsonb_array_length(v_source.reference->'source_columns') <> 1 + OR v_source.reference->'target_columns' <> '["id"]'::jsonb + OR v_source.reference->>'source_relkind' <> 'r' + OR (v_source.reference->>'source_is_partition')::boolean + OR (v_source.reference->>'source_has_inheritance')::boolean + OR v_source.reference->>'target_relkind' <> 'r' + OR (v_source.reference->>'target_is_partition')::boolean + OR (v_source.reference->>'target_has_inheritance')::boolean + OR v_source.reference->>'delete_action' <> 'a' THEN + RAISE EXCEPTION + 'unsupported storage.objects foreign-key shape at %.% (constraint %); cleanup fails closed', + v_source.reference->>'source_schema', + v_source.reference->>'source_table', + v_source.reference->>'constraint_name'; + END IF; + END LOOP; + + -- The row locks serialize every immediate FK KEY SHARE check. They are + -- acquired before the reference scan so a writer that commits while cleanup + -- waits is visible to the following READ COMMITTED statements. + PERFORM object_row.id + FROM storage.objects AS object_row + JOIN jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text + ) ON candidate.object_id = object_row.id + ORDER BY object_row.id + FOR UPDATE OF object_row; + + SELECT count(*) + INTO v_existing_count + FROM storage.objects AS object_row + JOIN jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text + ) ON candidate.object_id = object_row.id; + + IF EXISTS ( + SELECT 1 + FROM storage.objects AS object_row + JOIN jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text + ) ON candidate.object_id = object_row.id + WHERE object_row.storage_provider IS DISTINCT FROM candidate.storage_provider + OR object_row.bucket_name IS DISTINCT FROM candidate.bucket_name + OR object_row.object_key IS DISTINCT FROM candidate.object_key + OR object_row.provider_version_id IS DISTINCT FROM candidate.provider_version_id + OR object_row.content_hash IS DISTINCT FROM candidate.content_hash + ) THEN + RAISE EXCEPTION + 'refusing to clean a storage object whose provider, bucket, key, provider version, or hash changed'; + END IF; + + FOR v_source IN + SELECT reference->>'source_schema' AS source_schema, + reference->>'source_table' AS source_table, + reference->>'constraint_name' AS constraint_name, + reference->'source_columns'->>0 AS source_column + FROM jsonb_array_elements(v_after) AS reference_rows(reference) + ORDER BY source_schema, source_table, constraint_name + LOOP + EXECUTE format( + 'SELECT EXISTS (' + 'SELECT 1 FROM %I.%I AS source_row ' + 'WHERE source_row.%I = ANY ($1))', + v_source.source_schema, + v_source.source_table, + v_source.source_column + ) + INTO v_reference_exists + USING v_candidate_ids; + + IF v_reference_exists THEN + RAISE EXCEPTION + 'storage object cleanup is referenced by %.%.% (constraint %)', + v_source.source_schema, + v_source.source_table, + v_source.source_column, + v_source.constraint_name; + END IF; + END LOOP; + + -- DELETE is deliberately before the external-object boundary. The caller's + -- transaction remains open while bytes are removed: an external failure rolls + -- this deletion back, while a trigger/FK rejection arrives before any bytes go. + RETURN QUERY + DELETE FROM storage.objects AS object_row + USING jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text + ) + WHERE object_row.id = candidate.object_id + AND object_row.storage_provider = candidate.storage_provider + AND object_row.bucket_name = candidate.bucket_name + AND object_row.object_key = candidate.object_key + AND object_row.provider_version_id = candidate.provider_version_id + AND object_row.content_hash = candidate.content_hash + RETURNING object_row.*; + + GET DIAGNOSTICS v_deleted_count = ROW_COUNT; + IF v_deleted_count IS DISTINCT FROM v_existing_count THEN + RAISE EXCEPTION 'a storage object changed before transactional cleanup deletion'; + END IF; + + SET CONSTRAINTS ALL IMMEDIATE; + RETURN; +END; +$cleanup$; + +REVOKE ALL ON FUNCTION storage.prepare_backtest_object_cleanup(jsonb) FROM PUBLIC; + +COMMENT ON FUNCTION storage.prepare_backtest_object_cleanup(jsonb) IS +'Transaction-scoped, SECURITY DEFINER compensation for exact unreferenced objects in the canonical backtest-results namespace. Source FK locks precede the storage.objects target lock; exact rows are deleted before the external-object boundary and remain rollbackable until the caller commits.'; diff --git a/db-migration/src/test/java/com/idea2strategy/backend/migration/BacktestObjectCleanupCapabilityMigrationIntegrationTest.java b/db-migration/src/test/java/com/idea2strategy/backend/migration/BacktestObjectCleanupCapabilityMigrationIntegrationTest.java new file mode 100644 index 00000000..510da970 --- /dev/null +++ b/db-migration/src/test/java/com/idea2strategy/backend/migration/BacktestObjectCleanupCapabilityMigrationIntegrationTest.java @@ -0,0 +1,187 @@ +package com.idea2strategy.backend.migration; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.UUID; +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.postgresql.PostgreSQLContainer; + +@Testcontainers(disabledWithoutDocker = true) +class BacktestObjectCleanupCapabilityMigrationIntegrationTest { + + @Container + static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer("postgres:16-alpine"); + + @TempDir + Path temporaryDirectory; + + @Test + void runtimeRoleCanOnlyDeleteExactUnreferencedBacktestObjectsThroughTheCapability() + throws Exception { + var centralDirectory = Path.of(getClass().getClassLoader().getResource("db/migration").toURI()); + var bundle = CanonicalMigrationBundleAssembler.assemble( + centralDirectory, java.util.List.of(), temporaryDirectory.resolve("bundle")); + Flyway.configure() + .dataSource(POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()) + .locations("filesystem:" + bundle.directory()) + .load() + .migrate(); + + try (var connection = POSTGRES.createConnection(""); + var statement = connection.createStatement()) { + assertCapabilityDefinition(statement); + + var referencedId = UUID.randomUUID(); + var removableId = UUID.randomUUID(); + var unownedId = UUID.randomUUID(); + insertObject(connection, referencedId, canonicalKey(referencedId)); + insertObject(connection, removableId, canonicalKey(removableId)); + insertObject(connection, unownedId, "pipeline-owned/" + unownedId + "/" + hash() + ".parquet"); + statement.execute("CREATE SCHEMA task5_future_unreadable"); + statement.execute(""" + CREATE TABLE task5_future_unreadable.object_refs ( + object_id uuid REFERENCES storage.objects(id) + ) + """); + statement.execute("INSERT INTO task5_future_unreadable.object_refs VALUES ('" + referencedId + "')"); + + statement.execute("SET ROLE idea2strategy_backtest"); + try (var privileges = statement.executeQuery(""" + SELECT + has_table_privilege(current_user, 'storage.objects', 'DELETE'), + has_table_privilege(current_user, + (SELECT relation.oid + FROM pg_catalog.pg_class relation + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = 'task5_future_unreadable' + AND relation.relname = 'object_refs'), + 'SELECT'), + has_function_privilege(current_user, + 'storage.prepare_backtest_object_cleanup(jsonb)', 'EXECUTE') + """)) { + assertTrue(privileges.next()); + assertFalse(privileges.getBoolean(1)); + assertFalse(privileges.getBoolean(2)); + assertTrue(privileges.getBoolean(3)); + } + var directDelete = assertThrows( + SQLException.class, + () -> statement.execute("DELETE FROM storage.objects WHERE id='" + removableId + "'")); + assertEquals("42501", directDelete.getSQLState()); + + var referenced = assertThrows( + SQLException.class, + () -> cleanup(connection, referencedId, canonicalKey(referencedId))); + assertEquals("P0001", referenced.getSQLState(), referenced.getMessage()); + assertTrue(referenced.getMessage().contains("task5_future_unreadable.object_refs")); + + var unowned = assertThrows( + SQLException.class, + () -> cleanup( + connection, + unownedId, + "pipeline-owned/" + unownedId + "/" + hash() + ".parquet")); + assertEquals("P0001", unowned.getSQLState(), unowned.getMessage()); + assertTrue(unowned.getMessage().contains("canonical backtest")); + + assertEquals(removableId, cleanup(connection, removableId, canonicalKey(removableId))); + statement.execute("RESET ROLE"); + + assertEquals(1, objectCount(connection, referencedId)); + assertEquals(0, objectCount(connection, removableId)); + assertEquals(1, objectCount(connection, unownedId)); + } + } + + private static void assertCapabilityDefinition(Statement statement) throws SQLException { + try (var definition = statement.executeQuery(""" + SELECT procedure.prosecdef, + procedure.proowner::regrole::text AS owner_name, + procedure.proconfig, + has_function_privilege('public', procedure.oid, 'EXECUTE'), + has_function_privilege('idea2strategy_backtest', procedure.oid, 'EXECUTE') + FROM pg_proc procedure + WHERE procedure.oid = + to_regprocedure('storage.prepare_backtest_object_cleanup(jsonb)') + """)) { + assertTrue(definition.next(), "the forward migration must install the cleanup capability"); + assertTrue(definition.getBoolean(1)); + assertNotEquals("idea2strategy_backtest", definition.getString(2)); + var settings = (String[]) definition.getArray(3).getArray(); + assertTrue(java.util.List.of(settings).contains("search_path=pg_catalog, pg_temp")); + assertTrue(java.util.List.of(settings).contains("lock_timeout=5s")); + assertFalse(definition.getBoolean(4)); + assertTrue(definition.getBoolean(5)); + } + } + + private static UUID cleanup(Connection connection, UUID id, String key) throws SQLException { + var payload = """ + [{"object_id":"%s","storage_provider":"S3_COMPATIBLE",\ + "bucket_name":"task5","object_key":"%s",\ + "provider_version_id":"version-1","content_hash":"%s"}] + """.formatted(id, key, hash()).replace("\n", ""); + try (PreparedStatement cleanup = connection.prepareStatement( + "SELECT id FROM storage.prepare_backtest_object_cleanup(?::jsonb)")) { + cleanup.setString(1, payload); + try (var rows = cleanup.executeQuery()) { + assertTrue(rows.next()); + var removed = rows.getObject(1, UUID.class); + assertFalse(rows.next()); + return removed; + } + } + } + + private static void insertObject(Connection connection, UUID id, String key) throws SQLException { + try (PreparedStatement insert = connection.prepareStatement(""" + INSERT INTO storage.objects + (id, status, storage_provider, bucket_name, object_key, + provider_version_id, content_hash, byte_size, file_format, + compression_codec, media_type, schema_version, row_count, + period_start, period_end, retention_policy_version) + VALUES (?, 'AVAILABLE', 'S3_COMPATIBLE', 'task5', ?, 'version-1', ?, 1, + 'PARQUET', 'UNCOMPRESSED', 'application/octet-stream', '1.0.0', 1, + '2026-01-01T00:00:00Z', '2026-01-01T00:00:01Z', 'v1') + """)) { + insert.setObject(1, id); + insert.setString(2, key); + insert.setString(3, hash()); + insert.executeUpdate(); + } + } + + private static int objectCount(Connection connection, UUID id) throws SQLException { + try (PreparedStatement count = connection.prepareStatement( + "SELECT count(*) FROM storage.objects WHERE id=?")) { + count.setObject(1, id); + try (var rows = count.executeQuery()) { + assertTrue(rows.next()); + return rows.getInt(1); + } + } + } + + private static String canonicalKey(UUID runId) { + return "backtest-results/" + runId + + "/TASK5_CLEANUP/week_start=2026-01-05/part=0001/" + hash() + ".parquet"; + } + + private static String hash() { + return "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + } +} diff --git a/db-migration/src/test/java/com/idea2strategy/backend/migration/CanonicalMigrationBundleAssemblerTest.java b/db-migration/src/test/java/com/idea2strategy/backend/migration/CanonicalMigrationBundleAssemblerTest.java index 0e209886..5312cadf 100644 --- a/db-migration/src/test/java/com/idea2strategy/backend/migration/CanonicalMigrationBundleAssemblerTest.java +++ b/db-migration/src/test/java/com/idea2strategy/backend/migration/CanonicalMigrationBundleAssemblerTest.java @@ -44,6 +44,7 @@ void assemblesOnlyOwnedCanonicalContributionsInGlobalVersionOrder() throws Excep "V20260825000000__backend_basic_strategy_execution_completion.sql", "V20260825000001__pipeline_basic_strategy_feature_catalog.sql", "V20260826010000__backend_bind_room_invitations_to_accounts.sql", + "V20260902000000__pipeline_backtest_object_cleanup_capability.sql", DatabaseAccessPolicy.RUNTIME_GRANTS_FILE), result.orderedFileNames()); assertTrue(Files.readString(result.directory().resolve(DatabaseAccessPolicy.RUNTIME_GRANTS_FILE)) @@ -52,6 +53,9 @@ void assemblesOnlyOwnedCanonicalContributionsInGlobalVersionOrder() throws Excep .contains("GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE \"trading\".\"execution_markers\" TO idea2strategy_trading")); assertTrue(Files.readString(result.directory().resolve(DatabaseAccessPolicy.RUNTIME_GRANTS_FILE)) .contains("GRANT SELECT, INSERT ON TABLE \"backtest\".\"run_input_pins\" TO idea2strategy_backend")); + assertTrue(Files.readString(result.directory().resolve(DatabaseAccessPolicy.RUNTIME_GRANTS_FILE)) + .contains("GRANT EXECUTE ON FUNCTION \"storage\".\"prepare_backtest_object_cleanup\"(jsonb) " + + "TO idea2strategy_backtest")); assertTrue(Files.exists(result.directory().resolve(CanonicalMigrationBundle.MANIFEST_FILE))); assertTrue(Files.exists(result.directory().resolve(CanonicalMigrationBundle.DIGEST_FILE))); } diff --git a/db-migration/src/test/java/com/idea2strategy/backend/migration/DatabaseAccessPolicyTest.java b/db-migration/src/test/java/com/idea2strategy/backend/migration/DatabaseAccessPolicyTest.java index 3925a6e2..b08893f1 100644 --- a/db-migration/src/test/java/com/idea2strategy/backend/migration/DatabaseAccessPolicyTest.java +++ b/db-migration/src/test/java/com/idea2strategy/backend/migration/DatabaseAccessPolicyTest.java @@ -419,6 +419,21 @@ void grantsTheBacktestRoleTheStorageObjectPromotionItsRegistrarPerforms() throws sql.contains("GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE \"storage\".\"objects\" " + "TO idea2strategy_backtest;"), "widening must not have handed the worker DELETE"); + assertTrue( + sql.contains("GRANT EXECUTE ON FUNCTION " + + "\"storage\".\"prepare_backtest_object_cleanup\"(jsonb) " + + "TO idea2strategy_backtest;"), + "cleanup must be exposed only as the narrow function capability"); + assertTrue( + sql.contains("REVOKE ALL ON FUNCTION " + + "\"storage\".\"prepare_backtest_object_cleanup\"(jsonb) FROM PUBLIC;")); + for (var role : List.of("backend", "batch", "trading", "pipeline")) { + assertFalse( + sql.contains("GRANT EXECUTE ON FUNCTION " + + "\"storage\".\"prepare_backtest_object_cleanup\"(jsonb) " + + "TO idea2strategy_" + role + ";"), + role + " must not receive the backtest cleanup capability"); + } // Exactly one storage statement for this role: widening a privilege must not widen the surface. assertEquals( 1, diff --git a/db-migration/src/test/java/com/idea2strategy/backend/migration/MigrationPolicyTest.java b/db-migration/src/test/java/com/idea2strategy/backend/migration/MigrationPolicyTest.java index 83d6b45e..0bd004ca 100644 --- a/db-migration/src/test/java/com/idea2strategy/backend/migration/MigrationPolicyTest.java +++ b/db-migration/src/test/java/com/idea2strategy/backend/migration/MigrationPolicyTest.java @@ -64,7 +64,8 @@ void verifiesTheCheckedInMigrationDirectoryAndBaselineChecksum() throws Exceptio "V1__initial_schema.sql", "V20260825000000__backend_basic_strategy_execution_completion.sql", "V20260825000001__pipeline_basic_strategy_feature_catalog.sql", - "V20260826010000__backend_bind_room_invitations_to_accounts.sql"), + "V20260826010000__backend_bind_room_invitations_to_accounts.sql", + "V20260902000000__pipeline_backtest_object_cleanup_capability.sql"), plan.orderedFileNames()); } From 8145313be991ae66961fa562e434c743dc8f111e Mon Sep 17 00:00:00 2001 From: Ryan Min Date: Wed, 2 Sep 2026 17:26:11 +0900 Subject: [PATCH 09/13] fix(migration): fence backtest artifact cleanup ownership --- .../migration/DatabaseAccessPolicy.java | 30 +- .../src/main/resources/db/migration/README.md | 59 ++ ...peline_bind_backtest_cleanup_ownership.sql | 990 ++++++++++++++++++ ...nupCapabilityMigrationIntegrationTest.java | 393 ++++++- ...CanonicalMigrationBundleAssemblerTest.java | 4 + .../migration/DatabaseAccessPolicyTest.java | 13 + .../migration/MigrationPolicyTest.java | 3 +- 7 files changed, 1457 insertions(+), 35 deletions(-) create mode 100644 db-migration/src/main/resources/db/migration/V20260902000001__pipeline_bind_backtest_cleanup_ownership.sql diff --git a/db-migration/src/main/java/com/idea2strategy/backend/migration/DatabaseAccessPolicy.java b/db-migration/src/main/java/com/idea2strategy/backend/migration/DatabaseAccessPolicy.java index 066c719f..621f1c29 100644 --- a/db-migration/src/main/java/com/idea2strategy/backend/migration/DatabaseAccessPolicy.java +++ b/db-migration/src/main/java/com/idea2strategy/backend/migration/DatabaseAccessPolicy.java @@ -12,6 +12,8 @@ public final class DatabaseAccessPolicy { public static final String RUNTIME_GRANTS_FILE = "R__database_runtime_grants.sql"; public static final String BACKTEST_OBJECT_CLEANUP_FUNCTION = "\"storage\".\"prepare_backtest_object_cleanup\"(jsonb)"; + public static final String BACKTEST_OBJECT_CLEANUP_REISSUE_FUNCTION = + "\"storage\".\"reissue_backtest_object_cleanup\"(jsonb, text)"; private static final String ROLE_PREFIX = "idea2strategy_"; private static final Pattern CREATE_TABLE = Pattern.compile( "(?i)CREATE\\s+TABLE\\s+(?:IF\\s+NOT\\s+EXISTS\\s+)?" @@ -236,21 +238,25 @@ public static String runtimeGrantSql(List migrationSql) { // storage.objects remains non-deletable by every application role. Backtest // compensation is a transaction-scoped, namespace-fenced SECURITY DEFINER // capability installed by a forward migration, so only EXECUTE is exposed. - sql.append("REVOKE ALL ON FUNCTION ") - .append(BACKTEST_OBJECT_CLEANUP_FUNCTION) - .append(" FROM PUBLIC;\n"); - for (var role : ApplicationRole.values()) { + for (var function : List.of( + BACKTEST_OBJECT_CLEANUP_FUNCTION, + BACKTEST_OBJECT_CLEANUP_REISSUE_FUNCTION)) { sql.append("REVOKE ALL ON FUNCTION ") - .append(BACKTEST_OBJECT_CLEANUP_FUNCTION) - .append(" FROM ") - .append(databaseRole(role)) + .append(function) + .append(" FROM PUBLIC;\n"); + for (var role : ApplicationRole.values()) { + sql.append("REVOKE ALL ON FUNCTION ") + .append(function) + .append(" FROM ") + .append(databaseRole(role)) + .append(";\n"); + } + sql.append("GRANT EXECUTE ON FUNCTION ") + .append(function) + .append(" TO ") + .append(databaseRole(ApplicationRole.BACKTEST)) .append(";\n"); } - sql.append("GRANT EXECUTE ON FUNCTION ") - .append(BACKTEST_OBJECT_CLEANUP_FUNCTION) - .append(" TO ") - .append(databaseRole(ApplicationRole.BACKTEST)) - .append(";\n"); return sql.toString(); } diff --git a/db-migration/src/main/resources/db/migration/README.md b/db-migration/src/main/resources/db/migration/README.md index 542c353b..86a069ac 100644 --- a/db-migration/src/main/resources/db/migration/README.md +++ b/db-migration/src/main/resources/db/migration/README.md @@ -34,3 +34,62 @@ The central assembler validates the immutable V1 checksum, migration naming, glo Every assembled bundle ends with generated repeatable migration `R__database_runtime_grants.sql`. `DatabaseAccessPolicy` remains its single source of truth. It creates credential-free group roles, revokes public application access, and grants only the required schema and table privileges. Environment-specific login roles and passwords remain deployment/bootstrap concerns and never appear in migration SQL. + +## `storage.objects` event trigger and RDS major upgrades + +`V20260902000001__pipeline_bind_backtest_cleanup_ownership.sql` installs the narrowly +scoped `storage_reject_unvalidated_object_fks` event trigger. It runs only after +`ALTER TABLE`, `CREATE TABLE`, or `CREATE TABLE AS`, and rejects only a command that +leaves an unvalidated foreign key targeting `storage.objects`. Unrelated DDL is not +blocked. + +PostgreSQL restricts event-trigger creation to superusers. The Development deployment +contract satisfies that requirement without relying on the application role: + +- `infra/terraform/environments/development/database.tf` creates the RDS master user + `idea2strategy_admin` with an AWS-managed master secret. +- `scripts/aws/development-database-bootstrap.sh` resolves that exact + `master_user_secret` and supplies its username and password to Flyway. +- The migration fails explicitly unless Flyway's current user is a PostgreSQL + superuser or a member of AWS RDS's `rds_superuser` role. + +AWS RDS requires event triggers to be removed before a major-version upgrade. During +the upgrade maintenance window, stop application and migration traffic, then run as +the RDS master user: + +```sql +DROP EVENT TRIGGER storage_reject_unvalidated_object_fks; +``` + +Immediately after the upgrade, first verify that no unsafe constraint was introduced: + +```sql +SELECT n.nspname AS source_schema, c.relname AS source_table, fk.conname +FROM pg_constraint AS fk +JOIN pg_class AS c ON c.oid = fk.conrelid +JOIN pg_namespace AS n ON n.oid = c.relnamespace +WHERE fk.contype = 'f' + AND fk.confrelid = 'storage.objects'::regclass + AND NOT fk.convalidated; +``` + +The result must be empty. Then recreate the trigger from the already-migrated function +and verify that it is enabled: + +```sql +CREATE EVENT TRIGGER storage_reject_unvalidated_object_fks +ON ddl_command_end +WHEN TAG IN ('ALTER TABLE', 'CREATE TABLE', 'CREATE TABLE AS') +EXECUTE FUNCTION storage.reject_unvalidated_storage_object_fks(); + +SELECT evtname, evtenabled +FROM pg_event_trigger +WHERE evtname = 'storage_reject_unvalidated_object_fks'; +``` + +Do not resume cleanup traffic unless the constraint query is empty and the trigger is +present with `evtenabled = 'O'`. See the PostgreSQL event-trigger privilege contract +and the AWS RDS major-upgrade event-trigger prerequisite: + +- +- diff --git a/db-migration/src/main/resources/db/migration/V20260902000001__pipeline_bind_backtest_cleanup_ownership.sql b/db-migration/src/main/resources/db/migration/V20260902000001__pipeline_bind_backtest_cleanup_ownership.sql new file mode 100644 index 00000000..540ec51d --- /dev/null +++ b/db-migration/src/main/resources/db/migration/V20260902000001__pipeline_bind_backtest_cleanup_ownership.sql @@ -0,0 +1,990 @@ +-- Backtest compensation is allowed only for rows that were atomically bound to +-- the live producing attempt when the storage row was first registered. This +-- forward migration also closes the ADD FOREIGN KEY ... NOT VALID race left by +-- V20260902000000 without rewriting that already-applied migration. + +CREATE TABLE storage.backtest_attempt_cleanup_capabilities ( + attempt_id uuid PRIMARY KEY + REFERENCES backtest.run_attempts(id) ON DELETE CASCADE, + run_id uuid NOT NULL + REFERENCES backtest.runs(id), + claim_token uuid NOT NULL, + capability_hash character varying(64) NOT NULL UNIQUE + CHECK (capability_hash ~ '^[0-9a-f]{64}$'), + created_at timestamp with time zone DEFAULT clock_timestamp() NOT NULL +); + +COMMENT ON TABLE storage.backtest_attempt_cleanup_capabilities IS +'Protected non-forgeable cleanup identity generated inside the transaction that creates an attempt. Application roles receive no direct read or mutation privilege.'; + +CREATE OR REPLACE FUNCTION storage.capture_backtest_attempt_cleanup_capability() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +AS $attempt_capability$ +DECLARE + v_capability text; +BEGIN + IF NEW.claim_token IS NULL THEN + RETURN NEW; + END IF; + + v_capability := encode(public.gen_random_bytes(32), 'hex'); + INSERT INTO storage.backtest_attempt_cleanup_capabilities( + attempt_id, + run_id, + claim_token, + capability_hash + ) VALUES ( + NEW.id, + NEW.run_id, + NEW.claim_token, + encode(public.digest(v_capability, 'sha256'), 'hex') + ); + PERFORM set_config( + 'idea2strategy.backtest_attempt_cleanup_capability', + v_capability, + true + ); + RETURN NEW; +END; +$attempt_capability$; + +REVOKE ALL ON FUNCTION storage.capture_backtest_attempt_cleanup_capability() FROM PUBLIC; + +CREATE TRIGGER capture_backtest_attempt_cleanup_capability +AFTER INSERT ON backtest.run_attempts +FOR EACH ROW +EXECUTE FUNCTION storage.capture_backtest_attempt_cleanup_capability(); + +CREATE TABLE storage.backtest_object_ownerships ( + object_id uuid PRIMARY KEY + REFERENCES storage.objects(id) ON DELETE CASCADE, + -- Do not add a redundant FK from run_id to backtest.runs. Terminal + -- publication holds that run row FOR UPDATE while this ownership row is + -- registered and committed through its narrow storage transaction; an FK + -- check on the second connection would self-deadlock. The producing-attempt + -- FK plus the definer trigger's exact attempt.run_id check proves the same + -- relationship without a second lock edge. + run_id uuid NOT NULL, + producing_attempt_id uuid NOT NULL + REFERENCES backtest.run_attempts(id), + producing_claim_token uuid NOT NULL, + cleanup_token_hash character varying(64) NOT NULL UNIQUE + CHECK (cleanup_token_hash ~ '^[0-9a-f]{64}$'), + created_at timestamp with time zone DEFAULT clock_timestamp() NOT NULL +); + +CREATE INDEX ix_backtest_object_ownerships_run_attempt + ON storage.backtest_object_ownerships(run_id, producing_attempt_id); + +COMMENT ON TABLE storage.backtest_object_ownerships IS +'Migration-owner ledger binding a newly registered canonical backtest object to its exact producing run and attempt. Application roles receive no direct mutation privilege.'; + +CREATE OR REPLACE FUNCTION storage.capture_backtest_object_ownership() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +AS $capture$ +DECLARE + v_attempt_id uuid; + v_attempt_text text; + v_claim_token uuid; + v_claim_text text; + v_attempt_capability text; + v_cleanup_token_hash text; + v_key_run_id uuid; + v_run_id uuid; + v_run_text text; +BEGIN + v_run_text := nullif(current_setting('idea2strategy.backtest_run_id', true), ''); + v_attempt_text := nullif(current_setting('idea2strategy.backtest_attempt_id', true), ''); + v_claim_text := nullif(current_setting('idea2strategy.backtest_claim_token', true), ''); + v_attempt_capability := nullif( + current_setting('idea2strategy.backtest_attempt_cleanup_capability', true), + '' + ); + v_cleanup_token_hash := nullif( + current_setting('idea2strategy.backtest_cleanup_token_hash', true), + '' + ); + + -- Other producers still register storage rows through their own policy. A + -- backtest row is cleanup-owned only when the engine deliberately supplies + -- all four transaction-local ownership values before the INSERT. + IF v_run_text IS NULL + AND v_attempt_text IS NULL + AND v_claim_text IS NULL + AND v_attempt_capability IS NULL + AND v_cleanup_token_hash IS NULL THEN + RETURN NEW; + END IF; + IF v_run_text IS NULL + OR v_attempt_text IS NULL + OR v_claim_text IS NULL + OR v_attempt_capability IS NULL + OR v_cleanup_token_hash IS NULL THEN + RAISE EXCEPTION + 'backtest object producer ownership requires run, attempt, claim, and cleanup token together'; + END IF; + IF v_cleanup_token_hash !~ '^[0-9a-f]{64}$' THEN + RAISE EXCEPTION + 'backtest object producer cleanup token hash is invalid'; + END IF; + + BEGIN + v_run_id := v_run_text::uuid; + v_attempt_id := v_attempt_text::uuid; + v_claim_token := v_claim_text::uuid; + v_key_run_id := substring( + NEW.object_key FROM + '^backtest-results/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/' + )::uuid; + EXCEPTION + WHEN invalid_text_representation THEN + RAISE EXCEPTION + 'backtest object producer ownership contains an invalid UUID'; + END; + + IF v_key_run_id IS NULL OR v_key_run_id IS DISTINCT FROM v_run_id THEN + RAISE EXCEPTION + 'backtest object producer run does not match its canonical object key'; + END IF; + + IF NOT EXISTS ( + SELECT 1 + FROM backtest.run_attempts AS attempt + WHERE attempt.id = v_attempt_id + AND attempt.run_id = v_run_id + AND attempt.claim_token = v_claim_token + AND attempt.status = 'RUNNING' + AND attempt.claim_expires_at > clock_timestamp() + AND EXISTS ( + SELECT 1 + FROM storage.backtest_attempt_cleanup_capabilities AS capability + WHERE capability.attempt_id = attempt.id + AND capability.run_id = attempt.run_id + AND capability.claim_token = attempt.claim_token + AND capability.capability_hash = encode( + public.digest(v_attempt_capability, 'sha256'), + 'hex' + ) + ) + ) THEN + RAISE EXCEPTION + 'backtest object producer ownership requires the current live attempt claim'; + END IF; + + INSERT INTO storage.backtest_object_ownerships( + object_id, + run_id, + producing_attempt_id, + producing_claim_token, + cleanup_token_hash + ) VALUES ( + NEW.id, + v_run_id, + v_attempt_id, + v_claim_token, + v_cleanup_token_hash + ); + RETURN NEW; +END; +$capture$; + +REVOKE ALL ON FUNCTION storage.capture_backtest_object_ownership() FROM PUBLIC; + +CREATE TRIGGER capture_backtest_object_ownership +AFTER INSERT ON storage.objects +FOR EACH ROW +EXECUTE FUNCTION storage.capture_backtest_object_ownership(); + +-- PostgreSQL reserves event-trigger administration to superusers. The deployed +-- Flyway identity is the RDS-managed main user (`idea2strategy_admin`), which AWS +-- assigns `rds_superuser`; local/rehearsal PostgreSQL uses a real superuser. Fail +-- here with an actionable contract message if deployment ever drifts to a weaker +-- migration identity rather than reaching CREATE EVENT TRIGGER ambiguously. +DO $event_trigger_capability$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles AS caller + WHERE caller.rolname = current_user + AND ( + caller.rolsuper + OR EXISTS ( + SELECT 1 + FROM pg_roles AS rds_role + WHERE rds_role.rolname = 'rds_superuser' + AND pg_has_role(caller.oid, rds_role.oid, 'MEMBER') + ) + ) + ) THEN + RAISE EXCEPTION + 'storage FK safety requires Flyway to use a PostgreSQL superuser or the RDS main/rds_superuser role'; + END IF; +END; +$event_trigger_capability$; + +CREATE OR REPLACE FUNCTION storage.reject_unvalidated_storage_object_fks() +RETURNS event_trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +AS $reject$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_event_trigger_ddl_commands() AS command + JOIN pg_constraint AS fk + ON fk.oid = command.objid + OR fk.conrelid = command.objid + WHERE fk.contype = 'f' + AND fk.confrelid = 'storage.objects'::regclass + AND NOT fk.convalidated + ) THEN + RAISE EXCEPTION + 'unvalidated foreign keys targeting storage.objects are forbidden'; + END IF; +END; +$reject$; + +REVOKE ALL ON FUNCTION storage.reject_unvalidated_storage_object_fks() FROM PUBLIC; + +CREATE EVENT TRIGGER storage_reject_unvalidated_object_fks +ON ddl_command_end +WHEN TAG IN ('ALTER TABLE', 'CREATE TABLE', 'CREATE TABLE AS') +EXECUTE FUNCTION storage.reject_unvalidated_storage_object_fks(); + +COMMENT ON EVENT TRIGGER storage_reject_unvalidated_object_fks IS +'Rejects only DDL that leaves an unvalidated foreign key targeting storage.objects. AWS RDS major upgrades require this event trigger to be dropped before upgrade and recreated immediately afterwards; see db migration README.'; + +CREATE OR REPLACE FUNCTION storage.reissue_backtest_object_cleanup( + p_candidate jsonb, + p_new_cleanup_token_hash text +) +RETURNS uuid +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +SET lock_timeout = '5s' +AS $reissue$ +DECLARE + v_attempt_capability text; + v_attempt_id uuid; + v_attempt_text text; + v_candidate record; + v_claim_text text; + v_claim_token uuid; + v_reference_exists boolean; + v_run_id uuid; + v_run_text text; + v_source record; +BEGIN + IF jsonb_typeof(p_candidate) IS DISTINCT FROM 'object' + OR ( + SELECT array_agg(key ORDER BY key) + FROM jsonb_object_keys(p_candidate) AS keys(key) + ) IS DISTINCT FROM ARRAY[ + 'bucket_name', + 'content_hash', + 'object_id', + 'object_key', + 'provider_version_id', + 'storage_provider' + ]::text[] THEN + RAISE EXCEPTION 'backtest object cleanup reissue candidate shape is invalid'; + END IF; + IF p_new_cleanup_token_hash !~ '^[0-9a-f]{64}$' THEN + RAISE EXCEPTION 'backtest object cleanup reissue token hash is invalid'; + END IF; + + SELECT * + INTO v_candidate + FROM jsonb_to_record(p_candidate) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text + ); + IF v_candidate.object_id IS NULL + OR nullif(v_candidate.storage_provider, '') IS NULL + OR nullif(v_candidate.bucket_name, '') IS NULL + OR nullif(v_candidate.object_key, '') IS NULL + OR nullif(v_candidate.provider_version_id, '') IS NULL + OR v_candidate.content_hash !~ '^[0-9a-f]{64}$' THEN + RAISE EXCEPTION 'backtest object cleanup reissue requires an exact object identity'; + END IF; + + v_run_text := nullif(current_setting('idea2strategy.backtest_run_id', true), ''); + v_attempt_text := nullif(current_setting('idea2strategy.backtest_attempt_id', true), ''); + v_claim_text := nullif(current_setting('idea2strategy.backtest_claim_token', true), ''); + v_attempt_capability := nullif( + current_setting('idea2strategy.backtest_attempt_cleanup_capability', true), + '' + ); + IF v_run_text IS NULL + OR v_attempt_text IS NULL + OR v_claim_text IS NULL + OR v_attempt_capability IS NULL THEN + RAISE EXCEPTION 'backtest object cleanup reissue requires current attempt capability context'; + END IF; + BEGIN + v_run_id := v_run_text::uuid; + v_attempt_id := v_attempt_text::uuid; + v_claim_token := v_claim_text::uuid; + EXCEPTION + WHEN invalid_text_representation THEN + RAISE EXCEPTION 'backtest object cleanup reissue attempt context is invalid'; + END; + + -- Lock the live successor attempt, rather than its run row. Attempt creation + -- always locks the run and then the previous latest attempt; therefore this + -- lock serializes expiry/reclaim and successor insertion without deadlocking + -- terminal publication, which already owns the run row on its outer UOW while + -- storage registration commits through a separate narrow UOW. + PERFORM caller.id + FROM backtest.run_attempts AS caller + JOIN storage.backtest_attempt_cleanup_capabilities AS capability + ON capability.attempt_id = caller.id + AND capability.run_id = caller.run_id + AND capability.claim_token = caller.claim_token + WHERE caller.id = v_attempt_id + AND caller.run_id = v_run_id + AND caller.claim_token = v_claim_token + AND caller.status = 'RUNNING' + AND caller.claim_expires_at > clock_timestamp() + AND capability.capability_hash = encode( + public.digest(v_attempt_capability, 'sha256'), + 'hex' + ) + AND NOT EXISTS ( + SELECT 1 + FROM backtest.run_attempts AS newer + WHERE newer.run_id = caller.run_id + AND newer.attempt_number > caller.attempt_number + ) + FOR UPDATE OF caller; + IF NOT FOUND THEN + RAISE EXCEPTION 'backtest object cleanup reissue claim is wrong, expired, or superseded'; + END IF; + + PERFORM object_row.id + FROM storage.objects AS object_row + WHERE object_row.id = v_candidate.object_id + AND object_row.storage_provider = v_candidate.storage_provider + AND object_row.bucket_name = v_candidate.bucket_name + AND object_row.object_key = v_candidate.object_key + AND object_row.provider_version_id = v_candidate.provider_version_id + AND object_row.content_hash = v_candidate.content_hash + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'backtest object cleanup reissue identity changed'; + END IF; + + IF NOT EXISTS ( + WITH RECURSIVE caller_lineage AS ( + SELECT attempt.id, attempt.previous_attempt_id, 0 AS depth + FROM backtest.run_attempts AS attempt + WHERE attempt.id = v_attempt_id + UNION ALL + SELECT predecessor.id, predecessor.previous_attempt_id, lineage.depth + 1 + FROM backtest.run_attempts AS predecessor + JOIN caller_lineage AS lineage + ON predecessor.id = lineage.previous_attempt_id + ) + SELECT 1 + FROM storage.backtest_object_ownerships AS ownership + JOIN backtest.run_attempts AS producer + ON producer.id = ownership.producing_attempt_id + AND producer.run_id = ownership.run_id + AND producer.claim_token = ownership.producing_claim_token + JOIN caller_lineage AS lineage + ON lineage.id = producer.id + WHERE ownership.object_id = v_candidate.object_id + AND ownership.run_id = v_run_id + AND ( + lineage.depth = 0 + OR producer.status IN ('FAILED', 'CANCELLED', 'SKIPPED') + ) + ) THEN + -- Exact immutable objects are intentionally reusable across runs. No + -- ownership mutation and a NULL result means this caller may publish a + -- reference, but receives no capability to compensate someone else's + -- bytes. This also covers pre-migration/unowned reconciliations. + RETURN NULL; + END IF; + + IF EXISTS ( + SELECT 1 + FROM storage.objects AS object_row + WHERE object_row.id = v_candidate.object_id + AND ( + object_row.legal_hold + OR ( + object_row.retention_until IS NOT NULL + AND object_row.retention_until > clock_timestamp() + ) + ) + ) THEN + RETURN NULL; + END IF; + + FOR v_source IN + SELECT source_namespace.nspname AS source_schema, + source.relname AS source_table, + source_column.attname AS source_column, + fk.conname AS constraint_name, + fk.convalidated AS validated, + cardinality(fk.conkey) AS source_column_count, + target_column.attname AS target_column, + source.relkind AS source_relkind, + source.relispartition AS source_is_partition + FROM pg_constraint AS fk + JOIN pg_class AS source ON source.oid = fk.conrelid + JOIN pg_namespace AS source_namespace + ON source_namespace.oid = source.relnamespace + JOIN pg_attribute AS source_column + ON source_column.attrelid = source.oid + AND source_column.attnum = fk.conkey[1] + JOIN pg_attribute AS target_column + ON target_column.attrelid = fk.confrelid + AND target_column.attnum = fk.confkey[1] + WHERE fk.contype = 'f' + AND fk.confrelid = 'storage.objects'::regclass + ORDER BY source_namespace.nspname, source.relname, fk.conname + LOOP + IF NOT v_source.validated THEN + RAISE EXCEPTION 'unvalidated foreign key targets storage.objects; reissue fails closed'; + END IF; + IF v_source.source_schema = 'storage' + AND v_source.source_table = 'backtest_object_ownerships' + AND v_source.source_column = 'object_id' THEN + CONTINUE; + END IF; + IF v_source.source_column_count <> 1 + OR v_source.target_column <> 'id' + OR v_source.source_relkind <> 'r' + OR v_source.source_is_partition THEN + RAISE EXCEPTION 'unsupported storage.objects foreign-key shape; reissue fails closed'; + END IF; + EXECUTE format( + 'SELECT EXISTS (SELECT 1 FROM %I.%I WHERE %I = $1)', + v_source.source_schema, + v_source.source_table, + v_source.source_column + ) INTO v_reference_exists USING v_candidate.object_id; + IF v_reference_exists THEN + -- A committed reference makes the existing object non-compensable. + -- Preserve it and do not rotate its cleanup capability. + RETURN NULL; + END IF; + END LOOP; + + UPDATE storage.backtest_object_ownerships + SET producing_attempt_id = v_attempt_id, + producing_claim_token = v_claim_token, + cleanup_token_hash = p_new_cleanup_token_hash + WHERE object_id = v_candidate.object_id; + IF NOT FOUND THEN + RAISE EXCEPTION 'backtest object cleanup reissue ownership vanished'; + END IF; + RETURN v_candidate.object_id; +END; +$reissue$; + +REVOKE ALL ON FUNCTION storage.reissue_backtest_object_cleanup(jsonb, text) FROM PUBLIC; + +CREATE OR REPLACE FUNCTION storage.prepare_backtest_object_cleanup(p_candidates jsonb) +RETURNS SETOF storage.objects +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +SET lock_timeout = '5s' +AS $cleanup$ +DECLARE + v_attempt_id uuid; + v_attempt_capability text; + v_attempt_text text; + v_candidate_count integer; + v_candidate_ids uuid[]; + v_claim_token uuid; + v_claim_text text; + v_deleted_count integer; + v_existing_count integer; + v_foreign_keys jsonb; + v_lock_acquired boolean := false; + v_lock_attempt integer; + v_reference_exists boolean; + v_relation record; + v_run_id uuid; + v_run_text text; + v_source record; + v_unique_count integer; +BEGIN + IF jsonb_typeof(p_candidates) IS DISTINCT FROM 'array' + OR jsonb_array_length(p_candidates) = 0 THEN + RAISE EXCEPTION 'backtest object cleanup candidates must be a non-empty JSON array'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM jsonb_array_elements(p_candidates) AS offered(candidate) + WHERE jsonb_typeof(offered.candidate) IS DISTINCT FROM 'object' + OR ( + SELECT array_agg(key ORDER BY key) + FROM jsonb_object_keys(offered.candidate) AS keys(key) + ) IS DISTINCT FROM ARRAY[ + 'bucket_name', + 'cleanup_token', + 'content_hash', + 'object_id', + 'object_key', + 'provider_version_id', + 'storage_provider' + ]::text[] + ) THEN + RAISE EXCEPTION 'backtest object cleanup candidate shape is invalid'; + END IF; + + SELECT count(*), count(DISTINCT candidate.object_id), + array_agg(candidate.object_id ORDER BY candidate.object_id) + INTO v_candidate_count, v_unique_count, v_candidate_ids + FROM jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text + ); + + IF v_candidate_count IS DISTINCT FROM v_unique_count THEN + RAISE EXCEPTION 'backtest object cleanup candidate ids must be unique'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text, + cleanup_token text + ) + WHERE candidate.object_id IS NULL + OR nullif(candidate.storage_provider, '') IS NULL + OR nullif(candidate.bucket_name, '') IS NULL + OR nullif(candidate.object_key, '') IS NULL + OR nullif(candidate.provider_version_id, '') IS NULL + OR candidate.content_hash !~ '^[0-9a-f]{64}$' + OR candidate.cleanup_token !~ '^[0-9a-f]{64}$' + OR NOT ( + ( + candidate.object_key ~ + '^backtest-results/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/[0-9a-f]{64}[.]json$' + AND candidate.object_key LIKE '%/' || candidate.content_hash || '.json' + ) + OR + ( + candidate.object_key ~ + '^backtest-results/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/[A-Z][A-Z0-9_]{0,49}/week_start=[0-9]{4}-[0-9]{2}-[0-9]{2}/part=[0-9]{4}/[0-9a-f]{64}[.]parquet$' + AND candidate.object_key LIKE '%/' || candidate.content_hash || '.parquet' + ) + ) + ) THEN + RAISE EXCEPTION + 'backtest object cleanup requires exact identities in the canonical backtest object namespace'; + END IF; + + -- Target-first locking prevents a new ADD FK from becoming visible between + -- catalog inspection and DELETE. Every durable table is then locked in a + -- DML-compatible mode using NOWAIT. If DDL already owns any possible source + -- and is waiting for the target, the PL/pgSQL exception subtransaction rolls + -- back immediately, releasing the target before a bounded retry. + FOR v_lock_attempt IN 1..12 LOOP + v_lock_acquired := false; + BEGIN + LOCK TABLE storage.objects + IN SHARE UPDATE EXCLUSIVE MODE NOWAIT; + + FOR v_relation IN + SELECT relation.oid AS relation_oid, + namespace.nspname AS relation_schema, + relation.relname AS relation_name + FROM pg_class AS relation + JOIN pg_namespace AS namespace + ON namespace.oid = relation.relnamespace + WHERE relation.relkind IN ('r', 'p') + AND relation.oid <> 'storage.objects'::regclass + AND namespace.nspname <> 'information_schema' + AND namespace.nspname !~ '^pg_(catalog|toast|temp_)' + ORDER BY namespace.nspname, relation.relname, relation.oid + LOOP + BEGIN + EXECUTE format( + 'LOCK TABLE %I.%I IN SHARE UPDATE EXCLUSIVE MODE NOWAIT', + v_relation.relation_schema, + v_relation.relation_name + ); + EXCEPTION + WHEN undefined_table THEN + RAISE lock_not_available USING MESSAGE = + 'a possible storage.objects foreign-key source changed during cleanup locking'; + END; + END LOOP; + v_lock_acquired := true; + EXCEPTION + WHEN lock_not_available OR deadlock_detected THEN + v_lock_acquired := false; + END; + + EXIT WHEN v_lock_acquired; + IF v_lock_attempt = 12 THEN + RAISE lock_not_available USING MESSAGE = + 'backtest object cleanup exhausted 12 bounded DDL-lock retries before external deletion'; + END IF; + PERFORM pg_sleep(least(0.01 * v_lock_attempt, 0.10)); + END LOOP; + + WITH foreign_keys AS ( + SELECT fk.oid AS constraint_oid, + fk.conname AS constraint_name, + source.oid AS source_oid, + source_namespace.nspname AS source_schema, + source.relname AS source_table, + source.relkind AS source_relkind, + source.relispartition AS source_is_partition, + EXISTS ( + SELECT 1 + FROM pg_inherits AS inheritance + WHERE inheritance.inhrelid = source.oid + OR inheritance.inhparent = source.oid + ) AS source_has_inheritance, + ARRAY( + SELECT source_column.attname + FROM unnest(fk.conkey) WITH ORDINALITY AS source_key(attnum, ordinality) + JOIN pg_attribute AS source_column + ON source_column.attrelid = source.oid + AND source_column.attnum = source_key.attnum + ORDER BY source_key.ordinality + ) AS source_columns, + target.relkind AS target_relkind, + target.relispartition AS target_is_partition, + EXISTS ( + SELECT 1 + FROM pg_inherits AS inheritance + WHERE inheritance.inhrelid = target.oid + OR inheritance.inhparent = target.oid + ) AS target_has_inheritance, + ARRAY( + SELECT target_column.attname + FROM unnest(fk.confkey) WITH ORDINALITY AS target_key(attnum, ordinality) + JOIN pg_attribute AS target_column + ON target_column.attrelid = target.oid + AND target_column.attnum = target_key.attnum + ORDER BY target_key.ordinality + ) AS target_columns, + fk.convalidated AS validated, + fk.condeferrable AS deferrable, + fk.confdeltype AS delete_action + FROM pg_constraint AS fk + JOIN pg_class AS source ON source.oid = fk.conrelid + JOIN pg_namespace AS source_namespace + ON source_namespace.oid = source.relnamespace + JOIN pg_class AS target ON target.oid = fk.confrelid + WHERE fk.contype = 'f' + AND fk.confrelid = 'storage.objects'::regclass + ) + SELECT coalesce( + jsonb_agg(to_jsonb(foreign_key) ORDER BY + foreign_key.source_schema, + foreign_key.source_table, + foreign_key.constraint_name, + foreign_key.constraint_oid), + '[]'::jsonb + ) + INTO v_foreign_keys + FROM foreign_keys AS foreign_key; + + FOR v_source IN + SELECT reference + FROM jsonb_array_elements(v_foreign_keys) AS reference_rows(reference) + LOOP + IF NOT (v_source.reference->>'validated')::boolean THEN + RAISE EXCEPTION + 'unvalidated foreign key %.% (constraint %) targets storage.objects; cleanup fails closed', + v_source.reference->>'source_schema', + v_source.reference->>'source_table', + v_source.reference->>'constraint_name'; + END IF; + + IF jsonb_array_length(v_source.reference->'source_columns') <> 1 + OR v_source.reference->'target_columns' <> '["id"]'::jsonb + OR v_source.reference->>'source_relkind' <> 'r' + OR (v_source.reference->>'source_is_partition')::boolean + OR (v_source.reference->>'source_has_inheritance')::boolean + OR v_source.reference->>'target_relkind' <> 'r' + OR (v_source.reference->>'target_is_partition')::boolean + OR (v_source.reference->>'target_has_inheritance')::boolean + OR ( + v_source.reference->>'delete_action' <> 'a' + AND NOT ( + v_source.reference->>'source_schema' = 'storage' + AND v_source.reference->>'source_table' = 'backtest_object_ownerships' + AND v_source.reference->'source_columns' = '["object_id"]'::jsonb + AND v_source.reference->>'delete_action' = 'c' + ) + ) THEN + RAISE EXCEPTION + 'unsupported storage.objects foreign-key shape at %.% (constraint %); cleanup fails closed', + v_source.reference->>'source_schema', + v_source.reference->>'source_table', + v_source.reference->>'constraint_name'; + END IF; + END LOOP; + + v_run_text := nullif(current_setting('idea2strategy.backtest_run_id', true), ''); + v_attempt_text := nullif(current_setting('idea2strategy.backtest_attempt_id', true), ''); + v_claim_text := nullif(current_setting('idea2strategy.backtest_claim_token', true), ''); + v_attempt_capability := nullif( + current_setting('idea2strategy.backtest_attempt_cleanup_capability', true), + '' + ); + IF v_run_text IS NULL + OR v_attempt_text IS NULL + OR v_claim_text IS NULL + OR v_attempt_capability IS NULL THEN + RAISE EXCEPTION + 'backtest object cleanup requires current producer ownership claim context'; + END IF; + BEGIN + v_run_id := v_run_text::uuid; + v_attempt_id := v_attempt_text::uuid; + v_claim_token := v_claim_text::uuid; + EXCEPTION + WHEN invalid_text_representation THEN + RAISE EXCEPTION + 'backtest object cleanup producer ownership claim contains an invalid UUID'; + END; + + -- Attempt creation/reclaim locks the run row first. Holding the same row + -- through external deletion fences a successor attempt from appearing after + -- the stale-predecessor check below. + PERFORM run_row.id + FROM backtest.runs AS run_row + WHERE run_row.id = v_run_id + FOR UPDATE; + + IF NOT FOUND OR NOT EXISTS ( + SELECT 1 + FROM backtest.run_attempts AS caller + WHERE caller.id = v_attempt_id + AND caller.run_id = v_run_id + AND caller.claim_token = v_claim_token + AND EXISTS ( + SELECT 1 + FROM storage.backtest_attempt_cleanup_capabilities AS capability + WHERE capability.attempt_id = caller.id + AND capability.run_id = caller.run_id + AND capability.claim_token = caller.claim_token + AND capability.capability_hash = encode( + public.digest(v_attempt_capability, 'sha256'), + 'hex' + ) + ) + AND ( + ( + caller.status = 'RUNNING' + AND caller.claim_expires_at > clock_timestamp() + ) + OR caller.status IN ('SUCCEEDED', 'FAILED', 'CANCELLED', 'SKIPPED') + ) + AND NOT EXISTS ( + SELECT 1 + FROM backtest.run_attempts AS successor + WHERE successor.run_id = caller.run_id + AND successor.attempt_number > caller.attempt_number + ) + ) THEN + RAISE EXCEPTION + 'backtest object cleanup claim is wrong, expired, stale, or superseded'; + END IF; + + PERFORM object_row.id + FROM storage.objects AS object_row + JOIN jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text + ) ON candidate.object_id = object_row.id + ORDER BY object_row.id + FOR UPDATE OF object_row; + + SELECT count(*) + INTO v_existing_count + FROM storage.objects AS object_row + JOIN jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text + ) ON candidate.object_id = object_row.id; + + IF EXISTS ( + SELECT 1 + FROM storage.objects AS object_row + JOIN jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text + ) ON candidate.object_id = object_row.id + WHERE object_row.storage_provider IS DISTINCT FROM candidate.storage_provider + OR object_row.bucket_name IS DISTINCT FROM candidate.bucket_name + OR object_row.object_key IS DISTINCT FROM candidate.object_key + OR object_row.provider_version_id IS DISTINCT FROM candidate.provider_version_id + OR object_row.content_hash IS DISTINCT FROM candidate.content_hash + ) THEN + RAISE EXCEPTION + 'refusing to clean a storage object whose provider, bucket, key, provider version, or hash changed'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM storage.objects AS object_row + JOIN jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text + ) ON candidate.object_id = object_row.id + WHERE object_row.legal_hold + OR object_row.retention_until > clock_timestamp() + ) THEN + RAISE EXCEPTION + 'backtest object cleanup is blocked by legal hold or unexpired retention'; + END IF; + + IF ( + WITH RECURSIVE caller_lineage AS ( + SELECT attempt.id, attempt.previous_attempt_id + FROM backtest.run_attempts AS attempt + WHERE attempt.id = v_attempt_id + UNION ALL + SELECT predecessor.id, predecessor.previous_attempt_id + FROM backtest.run_attempts AS predecessor + JOIN caller_lineage AS lineage + ON predecessor.id = lineage.previous_attempt_id + ) + SELECT count(*) + FROM storage.objects AS object_row + JOIN jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text, + cleanup_token text + ) ON candidate.object_id = object_row.id + JOIN storage.backtest_object_ownerships AS ownership + ON ownership.object_id = object_row.id + JOIN backtest.run_attempts AS producer + ON producer.id = ownership.producing_attempt_id + AND producer.run_id = ownership.run_id + AND producer.claim_token = ownership.producing_claim_token + WHERE ownership.run_id = v_run_id + AND ownership.cleanup_token_hash = encode( + public.digest(candidate.cleanup_token, 'sha256'), + 'hex' + ) + AND ownership.producing_attempt_id IN ( + SELECT lineage.id FROM caller_lineage AS lineage + ) + ) IS DISTINCT FROM v_existing_count THEN + RAISE EXCEPTION + 'backtest object cleanup candidate lacks exact producer ownership in the caller lineage'; + END IF; + + FOR v_source IN + SELECT reference->>'source_schema' AS source_schema, + reference->>'source_table' AS source_table, + reference->>'constraint_name' AS constraint_name, + reference->'source_columns'->>0 AS source_column + FROM jsonb_array_elements(v_foreign_keys) AS reference_rows(reference) + WHERE NOT ( + reference->>'source_schema' = 'storage' + AND reference->>'source_table' = 'backtest_object_ownerships' + AND reference->'source_columns' = '["object_id"]'::jsonb + ) + ORDER BY source_schema, source_table, constraint_name + LOOP + EXECUTE format( + 'SELECT EXISTS (' + 'SELECT 1 FROM %I.%I AS source_row ' + 'WHERE source_row.%I = ANY ($1))', + v_source.source_schema, + v_source.source_table, + v_source.source_column + ) + INTO v_reference_exists + USING v_candidate_ids; + + IF v_reference_exists THEN + RAISE EXCEPTION + 'storage object cleanup is referenced by %.%.% (constraint %)', + v_source.source_schema, + v_source.source_table, + v_source.source_column, + v_source.constraint_name; + END IF; + END LOOP; + + RETURN QUERY + DELETE FROM storage.objects AS object_row + USING jsonb_to_recordset(p_candidates) AS candidate( + object_id uuid, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text + ) + WHERE object_row.id = candidate.object_id + AND object_row.storage_provider = candidate.storage_provider + AND object_row.bucket_name = candidate.bucket_name + AND object_row.object_key = candidate.object_key + AND object_row.provider_version_id = candidate.provider_version_id + AND object_row.content_hash = candidate.content_hash + RETURNING object_row.*; + + GET DIAGNOSTICS v_deleted_count = ROW_COUNT; + IF v_deleted_count IS DISTINCT FROM v_existing_count THEN + RAISE EXCEPTION 'a storage object changed before transactional cleanup deletion'; + END IF; + + SET CONSTRAINTS ALL IMMEDIATE; + RETURN; +END; +$cleanup$; + +REVOKE ALL ON FUNCTION storage.prepare_backtest_object_cleanup(jsonb) FROM PUBLIC; + +COMMENT ON FUNCTION storage.prepare_backtest_object_cleanup(jsonb) IS +'Transaction-scoped cleanup for exact backtest objects owned by the caller attempt lineage. Target-first NOWAIT retries serialize every durable FK source; unvalidated constraints, legal holds, and live retention always fail before external deletion.'; diff --git a/db-migration/src/test/java/com/idea2strategy/backend/migration/BacktestObjectCleanupCapabilityMigrationIntegrationTest.java b/db-migration/src/test/java/com/idea2strategy/backend/migration/BacktestObjectCleanupCapabilityMigrationIntegrationTest.java index 510da970..d17ed84b 100644 --- a/db-migration/src/test/java/com/idea2strategy/backend/migration/BacktestObjectCleanupCapabilityMigrationIntegrationTest.java +++ b/db-migration/src/test/java/com/idea2strategy/backend/migration/BacktestObjectCleanupCapabilityMigrationIntegrationTest.java @@ -44,12 +44,18 @@ void runtimeRoleCanOnlyDeleteExactUnreferencedBacktestObjectsThroughTheCapabilit var statement = connection.createStatement()) { assertCapabilityDefinition(statement); + var runId = UUID.randomUUID(); + var owner = insertLiveOwner(connection, runId, null); var referencedId = UUID.randomUUID(); var removableId = UUID.randomUUID(); var unownedId = UUID.randomUUID(); - insertObject(connection, referencedId, canonicalKey(referencedId)); - insertObject(connection, removableId, canonicalKey(removableId)); - insertObject(connection, unownedId, "pipeline-owned/" + unownedId + "/" + hash() + ".parquet"); + var referencedToken = cleanupToken("1"); + var removableToken = cleanupToken("2"); + insertOwnedObject( + connection, owner, referencedId, canonicalKey(runId, "REFERENCED"), referencedToken); + insertOwnedObject( + connection, owner, removableId, canonicalKey(runId, "REMOVABLE"), removableToken); + insertObject(connection, unownedId, canonicalKey(runId, "UNOWNED")); statement.execute("CREATE SCHEMA task5_future_unreadable"); statement.execute(""" CREATE TABLE task5_future_unreadable.object_refs ( @@ -71,12 +77,21 @@ void runtimeRoleCanOnlyDeleteExactUnreferencedBacktestObjectsThroughTheCapabilit AND relation.relname = 'object_refs'), 'SELECT'), has_function_privilege(current_user, - 'storage.prepare_backtest_object_cleanup(jsonb)', 'EXECUTE') + 'storage.prepare_backtest_object_cleanup(jsonb)', 'EXECUTE'), + has_function_privilege(current_user, + 'storage.reissue_backtest_object_cleanup(jsonb,text)', 'EXECUTE'), + has_table_privilege(current_user, + 'storage.backtest_object_ownerships', 'SELECT'), + has_table_privilege(current_user, + 'storage.backtest_attempt_cleanup_capabilities', 'SELECT') """)) { assertTrue(privileges.next()); assertFalse(privileges.getBoolean(1)); assertFalse(privileges.getBoolean(2)); assertTrue(privileges.getBoolean(3)); + assertTrue(privileges.getBoolean(4)); + assertFalse(privileges.getBoolean(5)); + assertFalse(privileges.getBoolean(6)); } var directDelete = assertThrows( SQLException.class, @@ -85,7 +100,12 @@ void runtimeRoleCanOnlyDeleteExactUnreferencedBacktestObjectsThroughTheCapabilit var referenced = assertThrows( SQLException.class, - () -> cleanup(connection, referencedId, canonicalKey(referencedId))); + () -> cleanup( + connection, + owner, + referencedId, + canonicalKey(runId, "REFERENCED"), + referencedToken)); assertEquals("P0001", referenced.getSQLState(), referenced.getMessage()); assertTrue(referenced.getMessage().contains("task5_future_unreadable.object_refs")); @@ -93,17 +113,206 @@ void runtimeRoleCanOnlyDeleteExactUnreferencedBacktestObjectsThroughTheCapabilit SQLException.class, () -> cleanup( connection, + owner, unownedId, - "pipeline-owned/" + unownedId + "/" + hash() + ".parquet")); + canonicalKey(runId, "UNOWNED"), + cleanupToken("3"))); assertEquals("P0001", unowned.getSQLState(), unowned.getMessage()); - assertTrue(unowned.getMessage().contains("canonical backtest")); + assertTrue(unowned.getMessage().contains("producer ownership")); - assertEquals(removableId, cleanup(connection, removableId, canonicalKey(removableId))); + var wrongToken = assertThrows( + SQLException.class, + () -> cleanup( + connection, + owner, + removableId, + canonicalKey(runId, "REMOVABLE"), + cleanupToken("9"))); + assertEquals("P0001", wrongToken.getSQLState(), wrongToken.getMessage()); + assertTrue(wrongToken.getMessage().contains("producer ownership")); + + var extraCandidateField = candidate( + removableId, + canonicalKey(runId, "REMOVABLE"), + removableToken).replace("}", ",\"escalate\":true}"); + var adversarial = assertThrows( + SQLException.class, + () -> cleanupPayload(connection, owner, extraCandidateField)); + assertEquals("P0001", adversarial.getSQLState(), adversarial.getMessage()); + assertTrue(adversarial.getMessage().contains("shape is invalid")); + + assertEquals( + removableId, + cleanup( + connection, + owner, + removableId, + canonicalKey(runId, "REMOVABLE"), + removableToken)); statement.execute("RESET ROLE"); assertEquals(1, objectCount(connection, referencedId)); assertEquals(0, objectCount(connection, removableId)); assertEquals(1, objectCount(connection, unownedId)); + statement.execute("DROP SCHEMA task5_future_unreadable CASCADE"); + statement.execute("DELETE FROM storage.objects WHERE id IN ('" + + referencedId + "','" + unownedId + "')"); + } + } + + @Test + void runtimeRoleCannotDeleteCanonicalObjectWithoutDurableProducerOwnership() + throws Exception { + var centralDirectory = Path.of(getClass().getClassLoader().getResource("db/migration").toURI()); + var bundle = CanonicalMigrationBundleAssembler.assemble( + centralDirectory, java.util.List.of(), temporaryDirectory.resolve("ownership-bundle")); + Flyway.configure() + .dataSource(POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()) + .locations("filesystem:" + bundle.directory()) + .load() + .migrate(); + + var runId = UUID.randomUUID(); + var objectId = UUID.randomUUID(); + var objectKey = canonicalKey(runId, "UNOWNED_CANONICAL"); + try (var connection = POSTGRES.createConnection(""); + var statement = connection.createStatement()) { + var owner = insertLiveOwner(connection, runId, null); + insertObject(connection, objectId, objectKey); + statement.execute("SET ROLE idea2strategy_backtest"); + + var denied = assertThrows( + SQLException.class, + () -> cleanup(connection, owner, objectId, objectKey, cleanupToken("4"))); + assertEquals("P0001", denied.getSQLState(), denied.getMessage()); + assertTrue(denied.getMessage().contains("producer ownership"), denied.getMessage()); + + statement.execute("RESET ROLE"); + assertEquals(1, objectCount(connection, objectId)); + statement.execute("DELETE FROM storage.objects WHERE id='" + objectId + "'"); + } + } + + @Test + void cleanupRejectsEveryUnvalidatedForeignKeyBeforeDeletingCandidateRows() + throws Exception { + var centralDirectory = Path.of(getClass().getClassLoader().getResource("db/migration").toURI()); + var bundle = CanonicalMigrationBundleAssembler.assemble( + centralDirectory, java.util.List.of(), temporaryDirectory.resolve("unvalidated-bundle")); + Flyway.configure() + .dataSource(POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()) + .locations("filesystem:" + bundle.directory()) + .load() + .migrate(); + + var runId = UUID.randomUUID(); + var objectId = UUID.randomUUID(); + var objectKey = canonicalKey(runId, "UNVALIDATED"); + var cleanupToken = cleanupToken("5"); + var schema = "task5_unvalidated_" + objectId.toString().replace("-", ""); + try (var connection = POSTGRES.createConnection(""); + var statement = connection.createStatement()) { + var owner = insertLiveOwner(connection, runId, null); + insertOwnedObject(connection, owner, objectId, objectKey, cleanupToken); + statement.execute("CREATE SCHEMA \"" + schema + "\""); + statement.execute(""" + DO $$ BEGIN + IF EXISTS ( + SELECT 1 FROM pg_event_trigger + WHERE evtname = 'storage_reject_unvalidated_object_fks' + ) THEN + EXECUTE 'ALTER EVENT TRIGGER storage_reject_unvalidated_object_fks DISABLE'; + END IF; + END $$ + """); + statement.execute("CREATE TABLE \"" + schema + "\".object_refs (object_id uuid)"); + statement.execute("ALTER TABLE \"" + schema + "\".object_refs " + + "ADD CONSTRAINT object_refs_storage_fk FOREIGN KEY (object_id) " + + "REFERENCES storage.objects(id) NOT VALID"); + statement.execute(""" + DO $$ BEGIN + IF EXISTS ( + SELECT 1 FROM pg_event_trigger + WHERE evtname = 'storage_reject_unvalidated_object_fks' + ) THEN + EXECUTE 'ALTER EVENT TRIGGER storage_reject_unvalidated_object_fks ENABLE'; + END IF; + END $$ + """); + + statement.execute("SET ROLE idea2strategy_backtest"); + var denied = assertThrows( + SQLException.class, + () -> cleanup(connection, owner, objectId, objectKey, cleanupToken)); + assertEquals("P0001", denied.getSQLState(), denied.getMessage()); + assertTrue(denied.getMessage().contains("unvalidated"), denied.getMessage()); + statement.execute("RESET ROLE"); + assertEquals(1, objectCount(connection, objectId)); + } finally { + try (var connection = POSTGRES.createConnection(""); + var statement = connection.createStatement()) { + statement.execute(""" + DO $$ BEGIN + IF EXISTS ( + SELECT 1 FROM pg_event_trigger + WHERE evtname = 'storage_reject_unvalidated_object_fks' + ) THEN + EXECUTE 'ALTER EVENT TRIGGER storage_reject_unvalidated_object_fks ENABLE'; + END IF; + END $$ + """); + statement.execute("DROP SCHEMA IF EXISTS \"" + schema + "\" CASCADE"); + statement.execute("DELETE FROM storage.objects WHERE id='" + objectId + "'"); + } + } + } + + @Test + void eventTriggerRejectsOnlyUnvalidatedStorageObjectForeignKeys() throws Exception { + var centralDirectory = Path.of(getClass().getClassLoader().getResource("db/migration").toURI()); + var bundle = CanonicalMigrationBundleAssembler.assemble( + centralDirectory, java.util.List.of(), temporaryDirectory.resolve("event-trigger-bundle")); + Flyway.configure() + .dataSource(POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()) + .locations("filesystem:" + bundle.directory()) + .load() + .migrate(); + + var schema = "task5_event_" + UUID.randomUUID().toString().replace("-", ""); + try (var connection = POSTGRES.createConnection(""); + var statement = connection.createStatement()) { + statement.execute("CREATE SCHEMA \"" + schema + "\""); + statement.execute("CREATE TABLE \"" + schema + "\".parent_ids (id uuid PRIMARY KEY)"); + statement.execute("CREATE TABLE \"" + schema + "\".unrelated_refs (id uuid)"); + statement.execute("ALTER TABLE \"" + schema + "\".unrelated_refs " + + "ADD CONSTRAINT unrelated_not_valid FOREIGN KEY (id) " + + "REFERENCES \"" + schema + "\".parent_ids(id) NOT VALID"); + statement.execute("CREATE TABLE \"" + schema + "\".object_refs (object_id uuid)"); + + var rejected = assertThrows( + SQLException.class, + () -> statement.execute("ALTER TABLE \"" + schema + "\".object_refs " + + "ADD CONSTRAINT unsafe_storage_fk FOREIGN KEY (object_id) " + + "REFERENCES storage.objects(id) NOT VALID")); + assertEquals("P0001", rejected.getSQLState(), rejected.getMessage()); + assertTrue(rejected.getMessage().contains("unvalidated")); + + statement.execute("ALTER TABLE \"" + schema + "\".object_refs " + + "ADD CONSTRAINT validated_storage_fk FOREIGN KEY (object_id) " + + "REFERENCES storage.objects(id)"); + try (var trigger = statement.executeQuery(""" + SELECT event_trigger.evtenabled, + procedure.proowner::regrole::text AS owner_name + FROM pg_event_trigger event_trigger + JOIN pg_proc procedure ON procedure.oid = event_trigger.evtfoid + WHERE event_trigger.evtname = 'storage_reject_unvalidated_object_fks' + """)) { + assertTrue(trigger.next()); + assertEquals("O", trigger.getString(1)); + assertNotEquals("idea2strategy_backtest", trigger.getString(2)); + assertFalse(trigger.next()); + } + statement.execute("DROP SCHEMA \"" + schema + "\" CASCADE"); } } @@ -129,21 +338,155 @@ private static void assertCapabilityDefinition(Statement statement) throws SQLEx } } - private static UUID cleanup(Connection connection, UUID id, String key) throws SQLException { - var payload = """ + private static UUID cleanup( + Connection connection, + Owner owner, + UUID id, + String key, + String cleanupToken) throws SQLException { + return cleanupPayload(connection, owner, candidate(id, key, cleanupToken)); + } + + private static UUID cleanupPayload(Connection connection, Owner owner, String payload) + throws SQLException { + var oldAutoCommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + try { + setAttemptContext(connection, owner); + try (PreparedStatement cleanup = connection.prepareStatement( + "SELECT id FROM storage.prepare_backtest_object_cleanup(?::jsonb)")) { + cleanup.setString(1, payload); + try (var rows = cleanup.executeQuery()) { + assertTrue(rows.next()); + var removed = rows.getObject(1, UUID.class); + assertFalse(rows.next()); + connection.commit(); + return removed; + } + } + } catch (SQLException | RuntimeException error) { + connection.rollback(); + throw error; + } finally { + connection.setAutoCommit(oldAutoCommit); + } + } + + private static String candidate(UUID id, String key, String cleanupToken) { + return """ [{"object_id":"%s","storage_provider":"S3_COMPATIBLE",\ "bucket_name":"task5","object_key":"%s",\ - "provider_version_id":"version-1","content_hash":"%s"}] - """.formatted(id, key, hash()).replace("\n", ""); - try (PreparedStatement cleanup = connection.prepareStatement( - "SELECT id FROM storage.prepare_backtest_object_cleanup(?::jsonb)")) { - cleanup.setString(1, payload); - try (var rows = cleanup.executeQuery()) { - assertTrue(rows.next()); - var removed = rows.getObject(1, UUID.class); - assertFalse(rows.next()); - return removed; + "provider_version_id":"version-1","content_hash":"%s",\ + "cleanup_token":"%s"}] + """.formatted(id, key, hash(), cleanupToken).replace("\n", ""); + } + + private static Owner insertLiveOwner(Connection connection, UUID runId, UUID previousAttemptId) + throws SQLException { + var attemptId = UUID.randomUUID(); + var claimToken = UUID.randomUUID(); + var oldAutoCommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + try (var statement = connection.createStatement()) { + statement.execute("SET LOCAL session_replication_role = replica"); + statement.execute(""" + INSERT INTO backtest.runs ( + id,bot_id,owner_account_id,configuration_hash,status, + evaluation_start,evaluation_end,initial_cash_amount, + market_rules_version,accounting_rules_version,precision_rules_version, + fee_policy_id,slippage_rate_bps,buying_power_buffer_policy_id, + idempotency_key,queued_at,started_at,owner_anonymized_at,lane, + message_id,canonical_payload_hash,aggregate_sequence, + execution_policy_version,idempotency_scope + ) VALUES ( + '%s','%s',NULL,'sha256:%s','RUNNING', + '2026-01-01','2026-01-02',1000, + 'market:1','accounting:1','precision:1', + '%s',0,'%s','TASK5:%s',clock_timestamp(),clock_timestamp(), + clock_timestamp(),'BASIC','%s','sha256:%s',1,'policy:1','TASK5' + ) + """.formatted( + runId, + UUID.randomUUID(), + hash(), + UUID.randomUUID(), + UUID.randomUUID(), + UUID.randomUUID(), + UUID.randomUUID(), + hash())); + statement.execute("SET LOCAL session_replication_role = origin"); + statement.execute(""" + INSERT INTO backtest.run_attempts ( + id,run_id,attempt_number,worker_execution_key,status,started_at, + claim_token,worker_id,claimed_at,claim_expires_at,last_heartbeat_at, + previous_attempt_id + ) VALUES ( + '%s','%s',%d,'TASK5:%s','RUNNING',clock_timestamp(), + '%s','task5-owner',clock_timestamp(), + clock_timestamp()+interval '1 hour',clock_timestamp(),%s + ) + """.formatted( + attemptId, + runId, + previousAttemptId == null ? 1 : 2, + attemptId, + claimToken, + previousAttemptId == null ? "NULL" : "'" + previousAttemptId + "'")); + String capability; + try (var generated = statement.executeQuery( + "SELECT current_setting('idea2strategy.backtest_attempt_cleanup_capability')")) { + assertTrue(generated.next()); + capability = generated.getString(1); + assertEquals(64, capability.length()); + } + connection.commit(); + return new Owner(runId, attemptId, claimToken, capability); + } catch (SQLException | RuntimeException error) { + connection.rollback(); + throw error; + } finally { + connection.setAutoCommit(oldAutoCommit); + } + } + + private static void insertOwnedObject( + Connection connection, + Owner owner, + UUID id, + String key, + String cleanupToken) throws SQLException { + var oldAutoCommit = connection.getAutoCommit(); + connection.setAutoCommit(false); + try { + setAttemptContext(connection, owner); + try (PreparedStatement token = connection.prepareStatement( + "SELECT set_config('idea2strategy.backtest_cleanup_token_hash', " + + "encode(public.digest(?, 'sha256'), 'hex'), true)")) { + token.setString(1, cleanupToken); + token.execute(); } + insertObject(connection, id, key); + connection.commit(); + } catch (SQLException | RuntimeException error) { + connection.rollback(); + throw error; + } finally { + connection.setAutoCommit(oldAutoCommit); + } + } + + private static void setAttemptContext(Connection connection, Owner owner) throws SQLException { + try (PreparedStatement context = connection.prepareStatement(""" + SELECT set_config('idea2strategy.backtest_run_id', ?, true), + set_config('idea2strategy.backtest_attempt_id', ?, true), + set_config('idea2strategy.backtest_claim_token', ?, true), + set_config('idea2strategy.backtest_attempt_cleanup_capability', ?, true) + """)) { + context.setString(1, owner.runId().toString()); + context.setString(2, owner.attemptId().toString()); + context.setString(3, owner.claimToken().toString()); + context.setString(4, owner.capability()); + context.execute(); } } @@ -176,12 +519,18 @@ private static int objectCount(Connection connection, UUID id) throws SQLExcepti } } - private static String canonicalKey(UUID runId) { + private static String canonicalKey(UUID runId, String recordType) { return "backtest-results/" + runId - + "/TASK5_CLEANUP/week_start=2026-01-05/part=0001/" + hash() + ".parquet"; + + "/" + recordType + "/week_start=2026-01-05/part=0001/" + hash() + ".parquet"; + } + + private static String cleanupToken(String hexDigit) { + return hexDigit.repeat(64); } private static String hash() { return "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; } + + private record Owner(UUID runId, UUID attemptId, UUID claimToken, String capability) {} } diff --git a/db-migration/src/test/java/com/idea2strategy/backend/migration/CanonicalMigrationBundleAssemblerTest.java b/db-migration/src/test/java/com/idea2strategy/backend/migration/CanonicalMigrationBundleAssemblerTest.java index 5312cadf..eab9512d 100644 --- a/db-migration/src/test/java/com/idea2strategy/backend/migration/CanonicalMigrationBundleAssemblerTest.java +++ b/db-migration/src/test/java/com/idea2strategy/backend/migration/CanonicalMigrationBundleAssemblerTest.java @@ -45,6 +45,7 @@ void assemblesOnlyOwnedCanonicalContributionsInGlobalVersionOrder() throws Excep "V20260825000001__pipeline_basic_strategy_feature_catalog.sql", "V20260826010000__backend_bind_room_invitations_to_accounts.sql", "V20260902000000__pipeline_backtest_object_cleanup_capability.sql", + "V20260902000001__pipeline_bind_backtest_cleanup_ownership.sql", DatabaseAccessPolicy.RUNTIME_GRANTS_FILE), result.orderedFileNames()); assertTrue(Files.readString(result.directory().resolve(DatabaseAccessPolicy.RUNTIME_GRANTS_FILE)) @@ -56,6 +57,9 @@ void assemblesOnlyOwnedCanonicalContributionsInGlobalVersionOrder() throws Excep assertTrue(Files.readString(result.directory().resolve(DatabaseAccessPolicy.RUNTIME_GRANTS_FILE)) .contains("GRANT EXECUTE ON FUNCTION \"storage\".\"prepare_backtest_object_cleanup\"(jsonb) " + "TO idea2strategy_backtest")); + assertTrue(Files.readString(result.directory().resolve(DatabaseAccessPolicy.RUNTIME_GRANTS_FILE)) + .contains("GRANT EXECUTE ON FUNCTION \"storage\".\"reissue_backtest_object_cleanup\"(jsonb, text) " + + "TO idea2strategy_backtest")); assertTrue(Files.exists(result.directory().resolve(CanonicalMigrationBundle.MANIFEST_FILE))); assertTrue(Files.exists(result.directory().resolve(CanonicalMigrationBundle.DIGEST_FILE))); } diff --git a/db-migration/src/test/java/com/idea2strategy/backend/migration/DatabaseAccessPolicyTest.java b/db-migration/src/test/java/com/idea2strategy/backend/migration/DatabaseAccessPolicyTest.java index b08893f1..7f42315e 100644 --- a/db-migration/src/test/java/com/idea2strategy/backend/migration/DatabaseAccessPolicyTest.java +++ b/db-migration/src/test/java/com/idea2strategy/backend/migration/DatabaseAccessPolicyTest.java @@ -427,12 +427,25 @@ void grantsTheBacktestRoleTheStorageObjectPromotionItsRegistrarPerforms() throws assertTrue( sql.contains("REVOKE ALL ON FUNCTION " + "\"storage\".\"prepare_backtest_object_cleanup\"(jsonb) FROM PUBLIC;")); + assertTrue( + sql.contains("GRANT EXECUTE ON FUNCTION " + + "\"storage\".\"reissue_backtest_object_cleanup\"(jsonb, text) " + + "TO idea2strategy_backtest;"), + "successor recovery must be exposed only as the narrow reissue capability"); + assertTrue( + sql.contains("REVOKE ALL ON FUNCTION " + + "\"storage\".\"reissue_backtest_object_cleanup\"(jsonb, text) FROM PUBLIC;")); for (var role : List.of("backend", "batch", "trading", "pipeline")) { assertFalse( sql.contains("GRANT EXECUTE ON FUNCTION " + "\"storage\".\"prepare_backtest_object_cleanup\"(jsonb) " + "TO idea2strategy_" + role + ";"), role + " must not receive the backtest cleanup capability"); + assertFalse( + sql.contains("GRANT EXECUTE ON FUNCTION " + + "\"storage\".\"reissue_backtest_object_cleanup\"(jsonb, text) " + + "TO idea2strategy_" + role + ";"), + role + " must not receive the backtest reissue capability"); } // Exactly one storage statement for this role: widening a privilege must not widen the surface. assertEquals( diff --git a/db-migration/src/test/java/com/idea2strategy/backend/migration/MigrationPolicyTest.java b/db-migration/src/test/java/com/idea2strategy/backend/migration/MigrationPolicyTest.java index 0bd004ca..d4bece42 100644 --- a/db-migration/src/test/java/com/idea2strategy/backend/migration/MigrationPolicyTest.java +++ b/db-migration/src/test/java/com/idea2strategy/backend/migration/MigrationPolicyTest.java @@ -65,7 +65,8 @@ void verifiesTheCheckedInMigrationDirectoryAndBaselineChecksum() throws Exceptio "V20260825000000__backend_basic_strategy_execution_completion.sql", "V20260825000001__pipeline_basic_strategy_feature_catalog.sql", "V20260826010000__backend_bind_room_invitations_to_accounts.sql", - "V20260902000000__pipeline_backtest_object_cleanup_capability.sql"), + "V20260902000000__pipeline_backtest_object_cleanup_capability.sql", + "V20260902000001__pipeline_bind_backtest_cleanup_ownership.sql"), plan.orderedFileNames()); } From 5ae39cb4df27082c6af1cc0a439ac8ee866d4f7d Mon Sep 17 00:00:00 2001 From: Ryan Min Date: Wed, 2 Sep 2026 20:11:43 +0900 Subject: [PATCH 10/13] security: fence backtest runtime writes --- .../migration/DatabaseAccessPolicy.java | 45 +- .../src/main/resources/db/migration/README.md | 8 + ...backtest_narrow_runtime_attempt_writes.sql | 337 ++++++++++++ ...pipeline_narrow_backtest_object_writes.sql | 518 ++++++++++++++++++ ...iteCapabilityMigrationIntegrationTest.java | 399 ++++++++++++++ ...CanonicalMigrationBundleAssemblerTest.java | 8 + .../migration/DatabaseAccessPolicyTest.java | 109 +++- .../migration/MigrationPolicyTest.java | 4 +- 8 files changed, 1384 insertions(+), 44 deletions(-) create mode 100644 db-migration/src/main/resources/db/migration/V20260902000002__backtest_narrow_runtime_attempt_writes.sql create mode 100644 db-migration/src/main/resources/db/migration/V20260902000003__pipeline_narrow_backtest_object_writes.sql create mode 100644 db-migration/src/test/java/com/idea2strategy/backend/migration/BacktestRuntimeWriteCapabilityMigrationIntegrationTest.java diff --git a/db-migration/src/main/java/com/idea2strategy/backend/migration/DatabaseAccessPolicy.java b/db-migration/src/main/java/com/idea2strategy/backend/migration/DatabaseAccessPolicy.java index 621f1c29..6a101c16 100644 --- a/db-migration/src/main/java/com/idea2strategy/backend/migration/DatabaseAccessPolicy.java +++ b/db-migration/src/main/java/com/idea2strategy/backend/migration/DatabaseAccessPolicy.java @@ -14,6 +14,18 @@ public final class DatabaseAccessPolicy { "\"storage\".\"prepare_backtest_object_cleanup\"(jsonb)"; public static final String BACKTEST_OBJECT_CLEANUP_REISSUE_FUNCTION = "\"storage\".\"reissue_backtest_object_cleanup\"(jsonb, text)"; + public static final String BACKTEST_ATTEMPT_CLAIM_FUNCTION = + "\"backtest\".\"claim_run_attempt\"(uuid, text, text, bigint)"; + public static final String BACKTEST_ATTEMPT_HEARTBEAT_FUNCTION = + "\"backtest\".\"heartbeat_run_attempt\"(uuid, uuid, bigint)"; + public static final String BACKTEST_ATTEMPT_CLOSE_FUNCTION = + "\"backtest\".\"close_run_attempt\"(uuid, uuid, text, text, text, boolean)"; + public static final String BACKTEST_ATTEMPT_RECOVERY_FUNCTION = + "\"backtest\".\"recover_expired_run_attempt\"(uuid, text, text)"; + public static final String BACKTEST_OBJECT_REGISTER_FUNCTION = + "\"storage\".\"register_backtest_object\"(jsonb)"; + public static final String BACKTEST_OBJECT_TRANSITION_FUNCTION = + "\"storage\".\"transition_backtest_object\"(uuid, text, timestamp with time zone)"; private static final String ROLE_PREFIX = "idea2strategy_"; private static final Pattern CREATE_TABLE = Pattern.compile( "(?i)CREATE\\s+TABLE\\s+(?:IF\\s+NOT\\s+EXISTS\\s+)?" @@ -235,12 +247,19 @@ public static String runtimeGrantSql(List migrationSql) { .append(" TO ").append(roleName).append(";\n"); } } - // storage.objects remains non-deletable by every application role. Backtest - // compensation is a transaction-scoped, namespace-fenced SECURITY DEFINER - // capability installed by a forward migration, so only EXECUTE is exposed. + // Protected attempt and storage-object state is never mutated through a + // table-wide grant. Claim, heartbeat, close, recovery, staged registration, + // verification, reissue and compensation are narrow SECURITY DEFINER + // capabilities installed by forward migrations, so only EXECUTE is exposed. for (var function : List.of( BACKTEST_OBJECT_CLEANUP_FUNCTION, - BACKTEST_OBJECT_CLEANUP_REISSUE_FUNCTION)) { + BACKTEST_OBJECT_CLEANUP_REISSUE_FUNCTION, + BACKTEST_ATTEMPT_CLAIM_FUNCTION, + BACKTEST_ATTEMPT_HEARTBEAT_FUNCTION, + BACKTEST_ATTEMPT_CLOSE_FUNCTION, + BACKTEST_ATTEMPT_RECOVERY_FUNCTION, + BACKTEST_OBJECT_REGISTER_FUNCTION, + BACKTEST_OBJECT_TRANSITION_FUNCTION)) { sql.append("REVOKE ALL ON FUNCTION ") .append(function) .append(" FROM PUBLIC;\n"); @@ -386,6 +405,9 @@ private static MigrationOwner ownerFor(QualifiedTable table) { private static boolean allowsBacktest(Access access, String schema, String table) { if ("backtest".equals(schema)) { + if ("run_attempts".equals(table)) { + return access == Access.READ; + } return access == Access.READ || access == Access.INSERT || access == Access.UPDATE; } if (("strategy".equals(schema) || "market_data".equals(schema)) && access == Access.READ) { @@ -412,19 +434,12 @@ private static boolean allowsBacktest(Access access, String schema, String table return "outbox_consumer_receipts".equals(table) && (access == Access.READ || access == Access.INSERT || access == Access.UPDATE); } - // The worker registers its detail objects in two steps, because an object may not claim to be - // published before its bytes have been re-read: `register` inserts the row as STAGED, and - // `mark_available` promotes that same row to AVAILABLE once the checksum verifies. `quarantine` - // is the third statement, recording a verification failure against the row that already exists. - // Both transitions are `UPDATE storage.objects SET status = ...` in the engine's repository, so - // INSERT alone stops a run after it has written its bytes — which is what failed INT03 run - // 9095f2a3 five times, with SELECT and INSERT held and UPDATE denied on the deployed role. - // - // DELETE stays out: a storage row is the identity of bytes that exist, and the worker never - // retracts one. Corruption is recorded by moving the row to QUARANTINED, not by removing it. + // Registration and status transitions now execute through attempt-fenced + // functions. SELECT is sufficient for reconciliation and result reads; a + // table-wide INSERT/UPDATE would let this role forge AVAILABLE evidence. return "storage".equals(schema) && "objects".equals(table) - && (access == Access.READ || access == Access.INSERT || access == Access.UPDATE); + && access == Access.READ; } private static boolean allowsPipeline(Access access, String schema, String table) { diff --git a/db-migration/src/main/resources/db/migration/README.md b/db-migration/src/main/resources/db/migration/README.md index 86a069ac..49072c42 100644 --- a/db-migration/src/main/resources/db/migration/README.md +++ b/db-migration/src/main/resources/db/migration/README.md @@ -33,6 +33,14 @@ The central assembler validates the immutable V1 checksum, migration naming, glo Every assembled bundle ends with generated repeatable migration `R__database_runtime_grants.sql`. `DatabaseAccessPolicy` remains its single source of truth. It creates credential-free group roles, revokes public application access, and grants only the required schema and table privileges. +`V20260902000002__backtest_narrow_runtime_attempt_writes.sql` and +`V20260902000003__pipeline_narrow_backtest_object_writes.sql` remove direct +`run_attempts` and `storage.objects` mutation from the backtest runtime role. Claim, +heartbeat, close, recovery, registration, verification, cleanup, and immediate-successor +reconciliation are exposed only through attempt-fenced function capabilities. Existing +provider bytes registered without a database row remain deliberately unowned, and a +later descendant cannot adopt an older ancestor's artifact. + Environment-specific login roles and passwords remain deployment/bootstrap concerns and never appear in migration SQL. ## `storage.objects` event trigger and RDS major upgrades diff --git a/db-migration/src/main/resources/db/migration/V20260902000002__backtest_narrow_runtime_attempt_writes.sql b/db-migration/src/main/resources/db/migration/V20260902000002__backtest_narrow_runtime_attempt_writes.sql new file mode 100644 index 00000000..e7ffd529 --- /dev/null +++ b/db-migration/src/main/resources/db/migration/V20260902000002__backtest_narrow_runtime_attempt_writes.sql @@ -0,0 +1,337 @@ +-- Attempt rows are protected ownership evidence. Runtime workers may claim, +-- heartbeat, close, and recover them only through these fenced capabilities. + +CREATE FUNCTION backtest.claim_run_attempt( + p_run_id uuid, + p_worker_id text, + p_execution_key text, + p_lease_milliseconds bigint +) +RETURNS SETOF backtest.run_attempts +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +AS $claim$ +DECLARE + v_attempt_id uuid; + v_attempt_key text; + v_claim_token uuid; + v_inserted backtest.run_attempts%ROWTYPE; + v_latest backtest.run_attempts%ROWTYPE; + v_next_number integer := 1; + v_now timestamp with time zone := clock_timestamp(); + v_run record; +BEGIN + IF nullif(btrim(p_worker_id), '') IS NULL + OR nullif(btrim(p_execution_key), '') IS NULL THEN + RAISE EXCEPTION 'worker id and execution key must not be blank'; + END IF; + IF p_lease_milliseconds <= 0 OR p_lease_milliseconds > 86400000 THEN + RAISE EXCEPTION 'attempt lease must be between one millisecond and one day'; + END IF; + + SELECT run_row.status, run_row.cancellation_requested_at + INTO v_run + FROM backtest.runs AS run_row + WHERE run_row.id = p_run_id + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'backtest run not found: %', p_run_id; + END IF; + IF v_run.status IN ('COMPLETED', 'FAILED', 'CANCELLED', 'UNAVAILABLE') + OR v_run.cancellation_requested_at IS NOT NULL THEN + RETURN; + END IF; + + SELECT attempt.* + INTO v_latest + FROM backtest.run_attempts AS attempt + WHERE attempt.run_id = p_run_id + ORDER BY attempt.attempt_number DESC + LIMIT 1 + FOR UPDATE; + + IF FOUND THEN + v_next_number := v_latest.attempt_number + 1; + IF v_latest.status IN ('SUCCEEDED', 'CANCELLED', 'SKIPPED') THEN + RETURN; + END IF; + IF v_latest.status = 'FAILED' + AND v_latest.terminal_reason_code NOT IN ('LEASE_EXPIRED', 'RETRY_RELEASED') THEN + RETURN; + END IF; + IF v_latest.status = 'RUNNING' THEN + IF v_latest.claim_expires_at IS NOT NULL + AND v_latest.claim_expires_at > v_now THEN + RETURN; + END IF; + UPDATE backtest.run_attempts AS attempt + SET status = 'FAILED', + completed_at = v_now, + failure_code = 'LEASE_EXPIRED', + terminal_reason_code = 'LEASE_EXPIRED' + WHERE attempt.id = v_latest.id + AND attempt.claim_token = v_latest.claim_token + AND attempt.status = 'RUNNING' + AND attempt.claim_expires_at <= v_now; + IF NOT FOUND THEN + RAISE EXCEPTION 'expired attempt was reclaimed concurrently'; + END IF; + END IF; + END IF; + + v_attempt_id := public.gen_random_uuid(); + v_claim_token := public.gen_random_uuid(); + v_attempt_key := p_execution_key || ':' || v_next_number; + IF length(v_attempt_key) > 160 THEN + RAISE EXCEPTION 'versioned worker execution key exceeds varchar(160)'; + END IF; + + INSERT INTO backtest.run_attempts( + id, + run_id, + attempt_number, + worker_execution_key, + status, + claim_token, + worker_id, + claimed_at, + claim_expires_at, + last_heartbeat_at, + previous_attempt_id, + started_at + ) VALUES ( + v_attempt_id, + p_run_id, + v_next_number, + v_attempt_key, + 'RUNNING', + v_claim_token, + p_worker_id, + v_now, + v_now + make_interval(secs => p_lease_milliseconds::double precision / 1000.0), + v_now, + CASE WHEN v_next_number = 1 THEN NULL ELSE v_latest.id END, + v_now + ) + ON CONFLICT DO NOTHING + RETURNING * INTO v_inserted; + IF NOT FOUND THEN + RAISE EXCEPTION 'attempt slot or execution key was claimed concurrently'; + END IF; + + UPDATE backtest.runs AS run_row + SET status = 'RUNNING', + started_at = coalesce(run_row.started_at, v_now) + WHERE run_row.id = p_run_id + AND run_row.status = 'QUEUED'; + + RETURN NEXT v_inserted; +END; +$claim$; + +REVOKE ALL ON FUNCTION backtest.claim_run_attempt(uuid, text, text, bigint) FROM PUBLIC; + +CREATE FUNCTION backtest.heartbeat_run_attempt( + p_attempt_id uuid, + p_claim_token uuid, + p_lease_milliseconds bigint +) +RETURNS SETOF backtest.run_attempts +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +AS $heartbeat$ +DECLARE + v_updated backtest.run_attempts%ROWTYPE; + v_now timestamp with time zone := clock_timestamp(); +BEGIN + IF p_lease_milliseconds <= 0 OR p_lease_milliseconds > 86400000 THEN + RAISE EXCEPTION 'attempt lease must be between one millisecond and one day'; + END IF; + UPDATE backtest.run_attempts AS attempt + SET last_heartbeat_at = v_now, + claim_expires_at = v_now + + make_interval(secs => p_lease_milliseconds::double precision / 1000.0) + WHERE attempt.id = p_attempt_id + AND attempt.claim_token = p_claim_token + AND attempt.status = 'RUNNING' + AND attempt.claim_expires_at > v_now + RETURNING attempt.* INTO v_updated; + IF FOUND THEN + RETURN NEXT v_updated; + END IF; +END; +$heartbeat$; + +REVOKE ALL ON FUNCTION backtest.heartbeat_run_attempt(uuid, uuid, bigint) FROM PUBLIC; + +CREATE FUNCTION backtest.close_run_attempt( + p_attempt_id uuid, + p_claim_token uuid, + p_status text, + p_terminal_reason_code text, + p_failure_code text, + p_requeue boolean +) +RETURNS SETOF backtest.run_attempts +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +AS $close$ +DECLARE + v_effective_failure text := p_failure_code; + v_effective_reason text := p_terminal_reason_code; + v_effective_status text := p_status; + v_existing backtest.run_attempts%ROWTYPE; + v_now timestamp with time zone := clock_timestamp(); + v_run record; + v_run_id uuid; + v_updated backtest.run_attempts%ROWTYPE; +BEGIN + IF p_status NOT IN ('SUCCEEDED', 'FAILED', 'CANCELLED', 'SKIPPED') + OR nullif(btrim(p_terminal_reason_code), '') IS NULL THEN + RAISE EXCEPTION 'attempt close requires a terminal status and reason'; + END IF; + IF p_requeue + AND (p_status <> 'FAILED' OR p_terminal_reason_code <> 'RETRY_RELEASED') THEN + RAISE EXCEPTION 'only RETRY_RELEASED may requeue a closed attempt'; + END IF; + + SELECT attempt.run_id + INTO v_run_id + FROM backtest.run_attempts AS attempt + WHERE attempt.id = p_attempt_id; + IF NOT FOUND THEN + RETURN; + END IF; + + SELECT run_row.status, run_row.cancellation_requested_at + INTO v_run + FROM backtest.runs AS run_row + WHERE run_row.id = v_run_id + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'backtest run not found: %', v_run_id; + END IF; + + PERFORM attempt.id + FROM backtest.run_attempts AS attempt + WHERE attempt.id = p_attempt_id + FOR UPDATE; + IF NOT FOUND THEN + RETURN; + END IF; + + IF v_run.cancellation_requested_at IS NOT NULL + AND p_status = 'SUCCEEDED' THEN + v_effective_status := 'CANCELLED'; + v_effective_reason := 'CANCELLED_BY_REQUEST'; + v_effective_failure := NULL; + END IF; + + UPDATE backtest.run_attempts AS attempt + SET status = v_effective_status::operations.work_status, + completed_at = v_now, + terminal_reason_code = v_effective_reason, + failure_code = v_effective_failure + WHERE attempt.id = p_attempt_id + AND attempt.claim_token = p_claim_token + AND attempt.status = 'RUNNING' + AND attempt.claim_expires_at > v_now + RETURNING attempt.* INTO v_updated; + + IF NOT FOUND THEN + SELECT attempt.* + INTO v_existing + FROM backtest.run_attempts AS attempt + WHERE attempt.id = p_attempt_id; + IF FOUND + AND v_existing.claim_token = p_claim_token + AND v_existing.status::text = v_effective_status THEN + RETURN NEXT v_existing; + END IF; + RETURN; + END IF; + + IF v_effective_status = 'CANCELLED' + AND p_status = 'SUCCEEDED' THEN + UPDATE backtest.runs AS run_row + SET status = 'CANCELLED', + cancelled_at = v_now, + completed_at = v_now + WHERE run_row.id = v_run_id + AND run_row.status = 'RUNNING'; + END IF; + + IF p_requeue THEN + UPDATE backtest.runs AS run_row + SET status = 'QUEUED' + WHERE run_row.id = v_run_id + AND run_row.status = 'RUNNING' + AND run_row.cancellation_requested_at IS NULL; + END IF; + RETURN NEXT v_updated; +END; +$close$; + +REVOKE ALL ON FUNCTION + backtest.close_run_attempt(uuid, uuid, text, text, text, boolean) + FROM PUBLIC; + +CREATE FUNCTION backtest.recover_expired_run_attempt( + p_attempt_id uuid, + p_status text, + p_reason text +) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +AS $recover$ +DECLARE + v_now timestamp with time zone := clock_timestamp(); + v_row_count integer; +BEGIN + IF p_status NOT IN ('FAILED', 'CANCELLED') + OR nullif(btrim(p_reason), '') IS NULL THEN + RAISE EXCEPTION 'expired-attempt recovery requires FAILED or CANCELLED and a reason'; + END IF; + UPDATE backtest.run_attempts AS attempt + SET status = p_status::operations.work_status, + completed_at = v_now, + failure_code = CASE WHEN p_status = 'CANCELLED' THEN NULL ELSE p_reason END, + terminal_reason_code = p_reason + WHERE attempt.id = p_attempt_id + AND attempt.status = 'RUNNING' + AND (attempt.claim_expires_at IS NULL OR attempt.claim_expires_at <= v_now) + AND ( + (p_status = 'CANCELLED' AND EXISTS ( + SELECT 1 + FROM backtest.runs AS run_row + WHERE run_row.id = attempt.run_id + AND run_row.cancellation_requested_at IS NOT NULL + )) + OR + (p_status = 'FAILED' AND EXISTS ( + SELECT 1 + FROM backtest.runs AS run_row + WHERE run_row.id = attempt.run_id + AND run_row.cancellation_requested_at IS NULL + )) + ) + AND NOT EXISTS ( + SELECT 1 + FROM backtest.run_attempts AS newer + WHERE newer.run_id = attempt.run_id + AND newer.attempt_number > attempt.attempt_number + ); + GET DIAGNOSTICS v_row_count = ROW_COUNT; + RETURN v_row_count; +END; +$recover$; + +REVOKE ALL ON FUNCTION backtest.recover_expired_run_attempt(uuid, text, text) FROM PUBLIC; + +COMMENT ON FUNCTION backtest.claim_run_attempt(uuid, text, text, bigint) IS +'Fenced database-time claim capability; runtime roles cannot insert or update run_attempts directly.'; diff --git a/db-migration/src/main/resources/db/migration/V20260902000003__pipeline_narrow_backtest_object_writes.sql b/db-migration/src/main/resources/db/migration/V20260902000003__pipeline_narrow_backtest_object_writes.sql new file mode 100644 index 00000000..e5d9024e --- /dev/null +++ b/db-migration/src/main/resources/db/migration/V20260902000003__pipeline_narrow_backtest_object_writes.sql @@ -0,0 +1,518 @@ +-- Runtime backtest workers may publish only through the staged-object lifecycle. +-- Keep the already-hardened +-- cleanup implementations from V20260902000001, but put a direct-successor gate +-- in front of them. Applied migrations remain untouched. + +CREATE OR REPLACE FUNCTION storage.capture_backtest_object_ownership() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +AS $capture$ +DECLARE + v_attempt_id uuid; + v_attempt_text text; + v_claim_token uuid; + v_claim_text text; + v_attempt_capability text; + v_cleanup_token_hash text; + v_key_run_id uuid; + v_run_id uuid; + v_run_text text; +BEGIN + v_run_text := nullif(current_setting('idea2strategy.backtest_run_id', true), ''); + v_attempt_text := nullif(current_setting('idea2strategy.backtest_attempt_id', true), ''); + v_claim_text := nullif(current_setting('idea2strategy.backtest_claim_token', true), ''); + v_attempt_capability := nullif( + current_setting('idea2strategy.backtest_attempt_cleanup_capability', true), + '' + ); + v_cleanup_token_hash := nullif( + current_setting('idea2strategy.backtest_cleanup_token_hash', true), + '' + ); + + -- Non-backtest producers supply no attempt context and remain outside this + -- ledger. A backtest registration always supplies all four attempt fields. + -- The cleanup token alone is optional: provider-reconciled bytes that have no + -- authoritative row are registered for reading, but deliberately stay unowned. + IF v_run_text IS NULL + AND v_attempt_text IS NULL + AND v_claim_text IS NULL + AND v_attempt_capability IS NULL + AND v_cleanup_token_hash IS NULL THEN + RETURN NEW; + END IF; + IF v_run_text IS NULL + OR v_attempt_text IS NULL + OR v_claim_text IS NULL + OR v_attempt_capability IS NULL THEN + RAISE EXCEPTION + 'backtest object registration requires run, attempt, claim, and attempt capability together'; + END IF; + IF v_cleanup_token_hash IS NOT NULL + AND v_cleanup_token_hash !~ '^[0-9a-f]{64}$' THEN + RAISE EXCEPTION 'backtest object producer cleanup token hash is invalid'; + END IF; + + BEGIN + v_run_id := v_run_text::uuid; + v_attempt_id := v_attempt_text::uuid; + v_claim_token := v_claim_text::uuid; + v_key_run_id := substring( + NEW.object_key FROM + '^backtest-results/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/' + )::uuid; + EXCEPTION + WHEN invalid_text_representation THEN + RAISE EXCEPTION 'backtest object producer ownership contains an invalid UUID'; + END; + + IF v_key_run_id IS NULL OR v_key_run_id IS DISTINCT FROM v_run_id THEN + RAISE EXCEPTION 'backtest object producer run does not match its canonical object key'; + END IF; + + IF NOT EXISTS ( + SELECT 1 + FROM backtest.run_attempts AS attempt + JOIN storage.backtest_attempt_cleanup_capabilities AS capability + ON capability.attempt_id = attempt.id + AND capability.run_id = attempt.run_id + AND capability.claim_token = attempt.claim_token + WHERE attempt.id = v_attempt_id + AND attempt.run_id = v_run_id + AND attempt.claim_token = v_claim_token + AND attempt.status = 'RUNNING' + AND attempt.claim_expires_at > clock_timestamp() + AND capability.capability_hash = encode( + public.digest(v_attempt_capability, 'sha256'), + 'hex' + ) + ) THEN + RAISE EXCEPTION 'backtest object registration requires the current live attempt claim'; + END IF; + + IF v_cleanup_token_hash IS NULL THEN + RETURN NEW; + END IF; + + INSERT INTO storage.backtest_object_ownerships( + object_id, + run_id, + producing_attempt_id, + producing_claim_token, + cleanup_token_hash + ) VALUES ( + NEW.id, + v_run_id, + v_attempt_id, + v_claim_token, + v_cleanup_token_hash + ); + RETURN NEW; +END; +$capture$; + +ALTER FUNCTION storage.reissue_backtest_object_cleanup(jsonb, text) + RENAME TO reissue_backtest_object_cleanup_recursive_legacy; +ALTER FUNCTION storage.prepare_backtest_object_cleanup(jsonb) + RENAME TO prepare_backtest_object_cleanup_recursive_legacy; + +REVOKE ALL ON FUNCTION + storage.reissue_backtest_object_cleanup_recursive_legacy(jsonb, text) + FROM PUBLIC; +REVOKE ALL ON FUNCTION + storage.prepare_backtest_object_cleanup_recursive_legacy(jsonb) + FROM PUBLIC; + +DO $revoke_legacy_cleanup$ +DECLARE + v_role text; +BEGIN + FOREACH v_role IN ARRAY ARRAY[ + 'idea2strategy_backend', + 'idea2strategy_batch', + 'idea2strategy_trading', + 'idea2strategy_backtest', + 'idea2strategy_pipeline' + ] LOOP + IF to_regrole(v_role) IS NOT NULL THEN + EXECUTE format( + 'REVOKE ALL ON FUNCTION storage.reissue_backtest_object_cleanup_recursive_legacy(jsonb, text) FROM %I', + v_role + ); + EXECUTE format( + 'REVOKE ALL ON FUNCTION storage.prepare_backtest_object_cleanup_recursive_legacy(jsonb) FROM %I', + v_role + ); + END IF; + END LOOP; +END; +$revoke_legacy_cleanup$; + +CREATE FUNCTION storage.reissue_backtest_object_cleanup( + p_candidate jsonb, + p_new_cleanup_token_hash text +) +RETURNS uuid +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +SET lock_timeout = '5s' +AS $reissue$ +DECLARE + v_attempt_id uuid; + v_attempt_text text; + v_candidate_id uuid; + v_owner_attempt_id uuid; + v_previous_attempt_id uuid; + v_run_id uuid; + v_run_text text; +BEGIN + -- Preserve the legacy function's closed input contract before inspecting the + -- protected ownership ledger. + IF jsonb_typeof(p_candidate) IS DISTINCT FROM 'object' + OR ( + SELECT array_agg(key ORDER BY key) + FROM jsonb_object_keys(p_candidate) AS keys(key) + ) IS DISTINCT FROM ARRAY[ + 'bucket_name', + 'content_hash', + 'object_id', + 'object_key', + 'provider_version_id', + 'storage_provider' + ]::text[] THEN + RAISE EXCEPTION 'backtest object cleanup reissue candidate shape is invalid'; + END IF; + IF p_new_cleanup_token_hash !~ '^[0-9a-f]{64}$' THEN + RAISE EXCEPTION 'backtest object cleanup reissue token hash is invalid'; + END IF; + + v_attempt_text := nullif(current_setting('idea2strategy.backtest_attempt_id', true), ''); + v_run_text := nullif(current_setting('idea2strategy.backtest_run_id', true), ''); + IF v_attempt_text IS NULL OR v_run_text IS NULL THEN + RAISE EXCEPTION 'backtest object cleanup reissue requires current attempt capability context'; + END IF; + BEGIN + v_attempt_id := v_attempt_text::uuid; + v_run_id := v_run_text::uuid; + v_candidate_id := (p_candidate->>'object_id')::uuid; + EXCEPTION + WHEN invalid_text_representation THEN + RAISE EXCEPTION 'backtest object cleanup reissue attempt context is invalid'; + END; + + SELECT attempt.previous_attempt_id + INTO v_previous_attempt_id + FROM backtest.run_attempts AS attempt + WHERE attempt.id = v_attempt_id + AND attempt.run_id = v_run_id; + + SELECT ownership.producing_attempt_id + INTO v_owner_attempt_id + FROM storage.backtest_object_ownerships AS ownership + WHERE ownership.object_id = v_candidate_id + AND ownership.run_id = v_run_id; + + -- An exact unowned object, an unrelated object, and an older ancestor are all + -- reusable immutable bytes, but none may be adopted for compensation. + IF v_owner_attempt_id IS NULL + OR v_owner_attempt_id NOT IN (v_attempt_id, v_previous_attempt_id) THEN + RETURN NULL; + END IF; + + RETURN storage.reissue_backtest_object_cleanup_recursive_legacy( + p_candidate, + p_new_cleanup_token_hash + ); +END; +$reissue$; + +REVOKE ALL ON FUNCTION storage.reissue_backtest_object_cleanup(jsonb, text) FROM PUBLIC; + +CREATE FUNCTION storage.prepare_backtest_object_cleanup(p_candidates jsonb) +RETURNS SETOF storage.objects +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +SET lock_timeout = '5s' +AS $cleanup$ +DECLARE + v_attempt_id uuid; + v_attempt_text text; + v_previous_attempt_id uuid; + v_run_id uuid; + v_run_text text; +BEGIN + IF jsonb_typeof(p_candidates) IS DISTINCT FROM 'array' THEN + RETURN QUERY + SELECT * + FROM storage.prepare_backtest_object_cleanup_recursive_legacy(p_candidates); + RETURN; + END IF; + + v_attempt_text := nullif(current_setting('idea2strategy.backtest_attempt_id', true), ''); + v_run_text := nullif(current_setting('idea2strategy.backtest_run_id', true), ''); + IF v_attempt_text IS NULL OR v_run_text IS NULL THEN + RAISE EXCEPTION 'backtest object cleanup requires current producer ownership claim context'; + END IF; + BEGIN + v_attempt_id := v_attempt_text::uuid; + v_run_id := v_run_text::uuid; + EXCEPTION + WHEN invalid_text_representation THEN + RAISE EXCEPTION 'backtest object cleanup producer ownership claim contains an invalid UUID'; + END; + + SELECT attempt.previous_attempt_id + INTO v_previous_attempt_id + FROM backtest.run_attempts AS attempt + WHERE attempt.id = v_attempt_id + AND attempt.run_id = v_run_id; + + IF EXISTS ( + SELECT 1 + FROM jsonb_to_recordset(p_candidates) AS candidate(object_id uuid) + JOIN storage.backtest_object_ownerships AS ownership + ON ownership.object_id = candidate.object_id + WHERE ownership.run_id = v_run_id + AND ownership.producing_attempt_id NOT IN ( + v_attempt_id, + v_previous_attempt_id + ) + ) THEN + RAISE EXCEPTION + 'backtest object cleanup candidate lacks exact producer ownership by the attempt or its immediate successor'; + END IF; + + RETURN QUERY + SELECT * + FROM storage.prepare_backtest_object_cleanup_recursive_legacy(p_candidates); +END; +$cleanup$; + +REVOKE ALL ON FUNCTION storage.prepare_backtest_object_cleanup(jsonb) FROM PUBLIC; + +CREATE FUNCTION storage.register_backtest_object(p_object jsonb) +RETURNS SETOF storage.objects +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +AS $register$ +DECLARE + v_object record; + v_registered storage.objects%ROWTYPE; +BEGIN + IF jsonb_typeof(p_object) IS DISTINCT FROM 'object' + OR ( + SELECT array_agg(key ORDER BY key) + FROM jsonb_object_keys(p_object) AS keys(key) + ) IS DISTINCT FROM ARRAY[ + 'bucket_name', 'byte_size', 'compression_codec', 'content_hash', + 'created_at', 'deleted_at', 'encryption_key_ref', 'file_format', 'id', + 'legal_hold', 'media_type', 'object_key', 'period_end', 'period_start', + 'provider_version_id', 'quarantined_at', 'retention_policy_version', + 'retention_until', 'row_count', 'schema_version', 'status', + 'storage_provider', 'superseded_at', 'verified_at' + ]::text[] THEN + RAISE EXCEPTION 'backtest storage registration document shape is invalid'; + END IF; + + SELECT * + INTO v_object + FROM jsonb_to_record(p_object) AS object_document( + id uuid, + status text, + storage_provider text, + bucket_name text, + object_key text, + provider_version_id text, + content_hash text, + byte_size bigint, + file_format text, + compression_codec text, + media_type text, + schema_version text, + row_count bigint, + period_start timestamp with time zone, + period_end timestamp with time zone, + encryption_key_ref text, + retention_policy_version text, + retention_until timestamp with time zone, + legal_hold boolean, + created_at timestamp with time zone, + verified_at timestamp with time zone, + quarantined_at timestamp with time zone, + superseded_at timestamp with time zone, + deleted_at timestamp with time zone + ); + IF v_object.id IS NULL + OR v_object.status IS DISTINCT FROM 'STAGED' + OR nullif(v_object.storage_provider, '') IS NULL + OR nullif(v_object.bucket_name, '') IS NULL + OR nullif(v_object.object_key, '') IS NULL + OR nullif(v_object.provider_version_id, '') IS NULL + OR v_object.content_hash !~ '^[0-9a-f]{64}$' + OR v_object.byte_size < 0 + OR nullif(v_object.file_format, '') IS NULL + OR nullif(v_object.compression_codec, '') IS NULL + OR nullif(v_object.media_type, '') IS NULL + OR nullif(v_object.schema_version, '') IS NULL + OR (v_object.row_count IS NOT NULL AND v_object.row_count < 0) + OR (v_object.period_start IS NOT NULL AND v_object.period_end < v_object.period_start) + OR nullif(v_object.retention_policy_version, '') IS NULL + OR v_object.legal_hold IS NULL + OR v_object.created_at IS NULL + OR v_object.verified_at IS NOT NULL + OR v_object.quarantined_at IS NOT NULL + OR v_object.superseded_at IS NOT NULL + OR v_object.deleted_at IS NOT NULL THEN + RAISE EXCEPTION 'backtest storage registration requires one exact STAGED object'; + END IF; + IF nullif(current_setting('idea2strategy.backtest_run_id', true), '') IS NULL + OR nullif(current_setting('idea2strategy.backtest_attempt_id', true), '') IS NULL + OR nullif(current_setting('idea2strategy.backtest_claim_token', true), '') IS NULL + OR nullif(current_setting('idea2strategy.backtest_attempt_cleanup_capability', true), '') IS NULL THEN + RAISE EXCEPTION 'backtest storage registration requires current attempt context'; + END IF; + + INSERT INTO storage.objects( + id, status, storage_provider, bucket_name, object_key, + provider_version_id, content_hash, byte_size, file_format, + compression_codec, media_type, schema_version, row_count, + period_start, period_end, encryption_key_ref, retention_policy_version, + retention_until, legal_hold, created_at, verified_at, quarantined_at, + superseded_at, deleted_at + ) VALUES ( + v_object.id, 'STAGED', v_object.storage_provider, v_object.bucket_name, + v_object.object_key, v_object.provider_version_id, v_object.content_hash, + v_object.byte_size, v_object.file_format, v_object.compression_codec, + v_object.media_type, v_object.schema_version, v_object.row_count, + v_object.period_start, v_object.period_end, v_object.encryption_key_ref, + v_object.retention_policy_version, v_object.retention_until, + v_object.legal_hold, v_object.created_at, NULL, NULL, NULL, NULL + ) + ON CONFLICT DO NOTHING + RETURNING * INTO v_registered; + IF FOUND THEN + RETURN NEXT v_registered; + END IF; +END; +$register$; + +REVOKE ALL ON FUNCTION storage.register_backtest_object(jsonb) FROM PUBLIC; + +CREATE FUNCTION storage.transition_backtest_object( + p_object_id uuid, + p_target text, + p_at timestamp with time zone +) +RETURNS SETOF storage.objects +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +AS $transition$ +DECLARE + v_attempt_capability text; + v_attempt_id uuid; + v_attempt_text text; + v_claim_token uuid; + v_claim_text text; + v_key_run_id uuid; + v_run_id uuid; + v_run_text text; + v_updated storage.objects%ROWTYPE; +BEGIN + IF p_target NOT IN ('AVAILABLE', 'QUARANTINED') OR p_at IS NULL THEN + RAISE EXCEPTION 'backtest object transition target is invalid'; + END IF; + v_run_text := nullif(current_setting('idea2strategy.backtest_run_id', true), ''); + v_attempt_text := nullif(current_setting('idea2strategy.backtest_attempt_id', true), ''); + v_claim_text := nullif(current_setting('idea2strategy.backtest_claim_token', true), ''); + v_attempt_capability := nullif( + current_setting('idea2strategy.backtest_attempt_cleanup_capability', true), + '' + ); + IF v_run_text IS NULL + OR v_attempt_text IS NULL + OR v_claim_text IS NULL + OR v_attempt_capability IS NULL THEN + RAISE EXCEPTION 'backtest object transition requires current attempt context'; + END IF; + BEGIN + v_run_id := v_run_text::uuid; + v_attempt_id := v_attempt_text::uuid; + v_claim_token := v_claim_text::uuid; + EXCEPTION + WHEN invalid_text_representation THEN + RAISE EXCEPTION 'backtest object transition attempt context is invalid'; + END; + + IF NOT EXISTS ( + SELECT 1 + FROM backtest.run_attempts AS attempt + JOIN storage.backtest_attempt_cleanup_capabilities AS capability + ON capability.attempt_id = attempt.id + AND capability.run_id = attempt.run_id + AND capability.claim_token = attempt.claim_token + WHERE attempt.id = v_attempt_id + AND attempt.run_id = v_run_id + AND attempt.claim_token = v_claim_token + AND attempt.status = 'RUNNING' + AND attempt.claim_expires_at > clock_timestamp() + AND capability.capability_hash = encode( + public.digest(v_attempt_capability, 'sha256'), + 'hex' + ) + ) THEN + RAISE EXCEPTION 'backtest object transition requires the current live attempt claim'; + END IF; + + SELECT object_row.* + INTO v_updated + FROM storage.objects AS object_row + WHERE object_row.id = p_object_id; + IF FOUND AND v_updated.status = p_target::storage.object_status THEN + RETURN NEXT v_updated; + RETURN; + END IF; + + SELECT substring( + object_row.object_key FROM + '^backtest-results/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/' + )::uuid + INTO v_key_run_id + FROM storage.objects AS object_row + WHERE object_row.id = p_object_id; + IF v_key_run_id IS NULL OR v_key_run_id IS DISTINCT FROM v_run_id THEN + RAISE EXCEPTION 'backtest object transition run does not match its canonical object key'; + END IF; + + IF p_target = 'AVAILABLE' THEN + UPDATE storage.objects AS object_row + SET status = 'AVAILABLE', verified_at = p_at + WHERE object_row.id = p_object_id + AND object_row.status = 'STAGED' + RETURNING object_row.* INTO v_updated; + ELSE + UPDATE storage.objects AS object_row + SET status = 'QUARANTINED', quarantined_at = p_at + WHERE object_row.id = p_object_id + AND object_row.status IN ('STAGED', 'AVAILABLE') + RETURNING object_row.* INTO v_updated; + END IF; + IF FOUND THEN + RETURN NEXT v_updated; + END IF; +END; +$transition$; + +REVOKE ALL ON FUNCTION + storage.transition_backtest_object(uuid, text, timestamp with time zone) + FROM PUBLIC; + +COMMENT ON FUNCTION storage.register_backtest_object(jsonb) IS +'Registers only a STAGED canonical backtest object for a live attempt; reconciled provider bytes may remain deliberately unowned.'; +COMMENT ON FUNCTION storage.prepare_backtest_object_cleanup(jsonb) IS +'Direct-producer or immediate-successor gate around the DDL-safe exact-version cleanup implementation.'; diff --git a/db-migration/src/test/java/com/idea2strategy/backend/migration/BacktestRuntimeWriteCapabilityMigrationIntegrationTest.java b/db-migration/src/test/java/com/idea2strategy/backend/migration/BacktestRuntimeWriteCapabilityMigrationIntegrationTest.java new file mode 100644 index 00000000..0f0286d6 --- /dev/null +++ b/db-migration/src/test/java/com/idea2strategy/backend/migration/BacktestRuntimeWriteCapabilityMigrationIntegrationTest.java @@ -0,0 +1,399 @@ +package com.idea2strategy.backend.migration; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.UUID; +import org.flywaydb.core.Flyway; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.postgresql.PostgreSQLContainer; + +@Testcontainers(disabledWithoutDocker = true) +class BacktestRuntimeWriteCapabilityMigrationIntegrationTest { + + @Container + static final PostgreSQLContainer POSTGRES = new PostgreSQLContainer("postgres:16-alpine"); + + @TempDir + Path temporaryDirectory; + + @Test + void runtimeRoleUsesFencedCapabilitiesInsteadOfForgingAttemptOrPublicationRows() + throws Exception { + var centralDirectory = Path.of(getClass().getClassLoader().getResource("db/migration").toURI()); + var bundle = CanonicalMigrationBundleAssembler.assemble( + centralDirectory, java.util.List.of(), temporaryDirectory.resolve("bundle")); + Flyway.configure() + .dataSource(POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()) + .locations("filesystem:" + bundle.directory()) + .load() + .migrate(); + + var runId = UUID.randomUUID(); + var objectId = UUID.randomUUID(); + var objectKey = "backtest-results/" + runId + + "/TASK6/week_start=2026-01-05/part=0001/" + hash() + ".parquet"; + insertQueuedRun(runId); + try (var connection = POSTGRES.createConnection("")) { + connection.setAutoCommit(true); + try (var statement = connection.createStatement()) { + statement.execute("SET ROLE idea2strategy_backtest"); + try (var privileges = statement.executeQuery(""" + SELECT + has_table_privilege(current_user, 'backtest.run_attempts', 'INSERT'), + has_table_privilege(current_user, 'backtest.run_attempts', 'UPDATE'), + has_table_privilege(current_user, 'storage.objects', 'INSERT'), + has_table_privilege(current_user, 'storage.objects', 'UPDATE') + """)) { + assertTrue(privileges.next()); + assertFalse(privileges.getBoolean(1)); + assertFalse(privileges.getBoolean(2)); + assertFalse(privileges.getBoolean(3)); + assertFalse(privileges.getBoolean(4)); + } + var directAttempt = assertThrows( + SQLException.class, + () -> statement.execute("INSERT INTO backtest.run_attempts " + + "(run_id,attempt_number,worker_execution_key,status,started_at) VALUES ('" + + runId + "',1,'FORGED','RUNNING',clock_timestamp())")); + assertEquals("42501", directAttempt.getSQLState()); + var directPublication = assertThrows( + SQLException.class, + () -> statement.execute("UPDATE storage.objects SET status='AVAILABLE' WHERE false")); + assertEquals("42501", directPublication.getSQLState()); + statement.execute("RESET ROLE"); + } + + connection.setAutoCommit(false); + UUID attemptId; + UUID claimToken; + String cleanupCapability; + try { + try (var statement = connection.createStatement()) { + statement.execute("SET LOCAL ROLE idea2strategy_backtest"); + } + try (PreparedStatement claim = connection.prepareStatement( + "SELECT id,claim_token FROM backtest.claim_run_attempt(?,?,?,?)")) { + claim.setObject(1, runId); + claim.setString(2, "task6-owner"); + claim.setString(3, "TASK6:FENCED"); + claim.setLong(4, 60_000L); + try (var rows = claim.executeQuery()) { + assertTrue(rows.next()); + attemptId = rows.getObject(1, UUID.class); + claimToken = rows.getObject(2, UUID.class); + assertFalse(rows.next()); + } + } + try (var statement = connection.createStatement(); + var capability = statement.executeQuery( + "SELECT current_setting('idea2strategy.backtest_attempt_cleanup_capability')")) { + assertTrue(capability.next()); + cleanupCapability = capability.getString(1); + assertEquals(64, cleanupCapability.length()); + } + connection.commit(); + } catch (SQLException | RuntimeException error) { + connection.rollback(); + throw error; + } + + try { + try (var statement = connection.createStatement()) { + statement.execute("SET LOCAL ROLE idea2strategy_backtest"); + } + setAttemptContext(connection, runId, attemptId, claimToken, cleanupCapability); + try (PreparedStatement token = connection.prepareStatement( + "SELECT set_config('idea2strategy.backtest_cleanup_token_hash', " + + "encode(public.digest(?, 'sha256'), 'hex'), true)")) { + token.setString(1, "a".repeat(64)); + token.execute(); + } + try (PreparedStatement register = connection.prepareStatement( + "SELECT id,status FROM storage.register_backtest_object(?::jsonb)")) { + register.setString(1, objectDocument(objectId, objectKey)); + try (var rows = register.executeQuery()) { + assertTrue(rows.next()); + assertEquals(objectId, rows.getObject(1, UUID.class)); + assertEquals("STAGED", rows.getString(2)); + assertFalse(rows.next()); + } + } + connection.commit(); + } catch (SQLException | RuntimeException error) { + connection.rollback(); + throw error; + } + + try { + try (var statement = connection.createStatement()) { + statement.execute("SET LOCAL ROLE idea2strategy_backtest"); + } + setAttemptContext(connection, runId, attemptId, claimToken, cleanupCapability); + try (PreparedStatement transition = connection.prepareStatement( + "SELECT status FROM storage.transition_backtest_object(?, 'AVAILABLE', clock_timestamp())")) { + transition.setObject(1, objectId); + try (var rows = transition.executeQuery()) { + assertTrue(rows.next()); + assertEquals("AVAILABLE", rows.getString(1)); + assertFalse(rows.next()); + } + } + try (PreparedStatement heartbeat = connection.prepareStatement( + "SELECT id FROM backtest.heartbeat_run_attempt(?,?,?)")) { + heartbeat.setObject(1, attemptId); + heartbeat.setObject(2, claimToken); + heartbeat.setLong(3, 60_000L); + try (var rows = heartbeat.executeQuery()) { + assertTrue(rows.next()); + assertEquals(attemptId, rows.getObject(1, UUID.class)); + assertFalse(rows.next()); + } + } + try (PreparedStatement close = connection.prepareStatement( + "SELECT status FROM backtest.close_run_attempt(?,?,?,?,?,?)")) { + close.setObject(1, attemptId); + close.setObject(2, claimToken); + close.setString(3, "SUCCEEDED"); + close.setString(4, "SUCCEEDED"); + close.setString(5, null); + close.setBoolean(6, false); + try (var rows = close.executeQuery()) { + assertTrue(rows.next()); + assertEquals("SUCCEEDED", rows.getString(1)); + assertFalse(rows.next()); + } + } + connection.commit(); + } catch (SQLException | RuntimeException error) { + connection.rollback(); + throw error; + } + } + + try (var connection = POSTGRES.createConnection(""); + var verify = connection.prepareStatement( + "SELECT o.status, count(ownership.object_id), attempt.status " + + "FROM storage.objects o " + + "LEFT JOIN storage.backtest_object_ownerships ownership " + + "ON ownership.object_id=o.id " + + "JOIN backtest.run_attempts attempt ON attempt.run_id=? " + + "WHERE o.id=? GROUP BY o.status,attempt.status")) { + verify.setObject(1, runId); + verify.setObject(2, objectId); + try (var rows = verify.executeQuery()) { + assertTrue(rows.next()); + assertEquals("AVAILABLE", rows.getString(1)); + assertEquals(1, rows.getInt(2)); + assertEquals("SUCCEEDED", rows.getString(3)); + assertFalse(rows.next()); + } + } + } + + @Test + void attemptCapabilitiesRejectUnfencedCancellationAndRecoveryOutcomes() + throws Exception { + var centralDirectory = Path.of(getClass().getClassLoader().getResource("db/migration").toURI()); + var bundle = CanonicalMigrationBundleAssembler.assemble( + centralDirectory, java.util.List.of(), temporaryDirectory.resolve("fence-bundle")); + Flyway.configure() + .dataSource(POSTGRES.getJdbcUrl(), POSTGRES.getUsername(), POSTGRES.getPassword()) + .locations("filesystem:" + bundle.directory()) + .load() + .migrate(); + + var cancelledRunId = UUID.randomUUID(); + insertQueuedRun(cancelledRunId); + UUID cancelledAttemptId; + try (var connection = POSTGRES.createConnection("")) { + connection.setAutoCommit(false); + try (var statement = connection.createStatement()) { + statement.execute("SET LOCAL ROLE idea2strategy_backtest"); + } + try (PreparedStatement claim = connection.prepareStatement( + "SELECT id FROM backtest.claim_run_attempt(?,?,?,?)")) { + claim.setObject(1, cancelledRunId); + claim.setString(2, "task6-fence-owner"); + claim.setString(3, "TASK6:FENCE-CANCEL"); + claim.setLong(4, 60_000L); + try (var rows = claim.executeQuery()) { + assertTrue(rows.next()); + cancelledAttemptId = rows.getObject(1, UUID.class); + } + } + connection.commit(); + } + try (var connection = POSTGRES.createConnection(""); + var cancellation = connection.prepareStatement( + "UPDATE backtest.runs SET cancellation_requested_at=clock_timestamp(), " + + "cancellation_reason_code='USER_CANCELLED' WHERE id=?")) { + cancellation.setObject(1, cancelledRunId); + assertEquals(1, cancellation.executeUpdate()); + } + try (var connection = POSTGRES.createConnection("")) { + connection.setAutoCommit(false); + try (var statement = connection.createStatement()) { + statement.execute("SET LOCAL ROLE idea2strategy_backtest"); + } + try (PreparedStatement close = connection.prepareStatement( + "SELECT status FROM backtest.close_run_attempt(?,?,?,?,?,?)")) { + close.setObject(1, cancelledAttemptId); + close.setObject(2, UUID.randomUUID()); + close.setString(3, "SUCCEEDED"); + close.setString(4, "SUCCEEDED"); + close.setString(5, null); + close.setBoolean(6, false); + try (var rows = close.executeQuery()) { + assertFalse(rows.next(), "a wrong fence must close nothing"); + } + } + connection.commit(); + } + try (var connection = POSTGRES.createConnection(""); + var verify = connection.prepareStatement( + "SELECT r.status,a.status FROM backtest.runs r " + + "JOIN backtest.run_attempts a ON a.run_id=r.id WHERE r.id=?")) { + verify.setObject(1, cancelledRunId); + try (var rows = verify.executeQuery()) { + assertTrue(rows.next()); + assertEquals("RUNNING", rows.getString(1)); + assertEquals("RUNNING", rows.getString(2)); + } + } + + var recoveryRunId = UUID.randomUUID(); + insertQueuedRun(recoveryRunId); + UUID recoveryAttemptId; + try (var connection = POSTGRES.createConnection("")) { + connection.setAutoCommit(false); + try (var statement = connection.createStatement()) { + statement.execute("SET LOCAL ROLE idea2strategy_backtest"); + } + try (PreparedStatement claim = connection.prepareStatement( + "SELECT id FROM backtest.claim_run_attempt(?,?,?,?)")) { + claim.setObject(1, recoveryRunId); + claim.setString(2, "task6-recovery-owner"); + claim.setString(3, "TASK6:FENCE-RECOVERY"); + claim.setLong(4, 60_000L); + try (var rows = claim.executeQuery()) { + assertTrue(rows.next()); + recoveryAttemptId = rows.getObject(1, UUID.class); + } + } + connection.commit(); + } + try (var connection = POSTGRES.createConnection(""); + var expire = connection.prepareStatement( + "UPDATE backtest.run_attempts SET " + + "started_at=clock_timestamp()-interval '3 minutes', " + + "claimed_at=clock_timestamp()-interval '3 minutes', " + + "last_heartbeat_at=clock_timestamp()-interval '2 minutes', " + + "claim_expires_at=clock_timestamp()-interval '1 minute' " + + "WHERE id=?")) { + expire.setObject(1, recoveryAttemptId); + assertEquals(1, expire.executeUpdate()); + } + try (var connection = POSTGRES.createConnection("")) { + connection.setAutoCommit(false); + try (var statement = connection.createStatement()) { + statement.execute("SET LOCAL ROLE idea2strategy_backtest"); + } + try (PreparedStatement recover = connection.prepareStatement( + "SELECT backtest.recover_expired_run_attempt(?, 'CANCELLED', 'CANCELLED_BY_REQUEST')")) { + recover.setObject(1, recoveryAttemptId); + try (var rows = recover.executeQuery()) { + assertTrue(rows.next()); + assertEquals(0, rows.getInt(1), "recovery cannot invent a cancellation request"); + } + } + connection.commit(); + } + } + + private void insertQueuedRun(UUID runId) throws SQLException { + try (var connection = POSTGRES.createConnection(""); + var statement = connection.createStatement()) { + connection.setAutoCommit(false); + try { + statement.execute("SET LOCAL session_replication_role = replica"); + statement.execute(""" + INSERT INTO backtest.runs ( + id,bot_id,owner_account_id,configuration_hash,status, + evaluation_start,evaluation_end,initial_cash_amount, + market_rules_version,accounting_rules_version,precision_rules_version, + fee_policy_id,slippage_rate_bps,buying_power_buffer_policy_id, + idempotency_key,queued_at,owner_anonymized_at,lane,message_id, + canonical_payload_hash,aggregate_sequence,execution_policy_version, + idempotency_scope + ) VALUES ( + '%s','%s',NULL,'sha256:%s','QUEUED','2026-01-01','2026-01-02',1000, + 'market:1','accounting:1','precision:1','%s',0,'%s','TASK6:%s', + clock_timestamp(),clock_timestamp(),'BASIC','%s','sha256:%s',1, + 'policy:1','TASK6' + ) + """.formatted( + runId, + UUID.randomUUID(), + hash(), + UUID.randomUUID(), + UUID.randomUUID(), + UUID.randomUUID(), + UUID.randomUUID(), + hash())); + connection.commit(); + } catch (SQLException | RuntimeException error) { + connection.rollback(); + throw error; + } + } + } + + private static void setAttemptContext( + Connection connection, + UUID runId, + UUID attemptId, + UUID claimToken, + String cleanupCapability) throws SQLException { + try (PreparedStatement context = connection.prepareStatement(""" + SELECT set_config('idea2strategy.backtest_run_id', ?, true), + set_config('idea2strategy.backtest_attempt_id', ?, true), + set_config('idea2strategy.backtest_claim_token', ?, true), + set_config('idea2strategy.backtest_attempt_cleanup_capability', ?, true) + """)) { + context.setString(1, runId.toString()); + context.setString(2, attemptId.toString()); + context.setString(3, claimToken.toString()); + context.setString(4, cleanupCapability); + context.execute(); + } + } + + private static String objectDocument(UUID objectId, String objectKey) { + return """ + {"id":"%s","status":"STAGED","storage_provider":"S3_COMPATIBLE",\ + "bucket_name":"task6","object_key":"%s","provider_version_id":"version-1",\ + "content_hash":"%s","byte_size":1,"file_format":"PARQUET",\ + "compression_codec":"UNCOMPRESSED","media_type":"application/octet-stream",\ + "schema_version":"1.0.0","row_count":1,\ + "period_start":"2026-01-01T00:00:00Z",\ + "period_end":"2026-01-01T00:00:01Z","encryption_key_ref":null,\ + "retention_policy_version":"v1","retention_until":null,"legal_hold":false,\ + "created_at":"2026-01-01T00:00:00Z","verified_at":null,\ + "quarantined_at":null,"superseded_at":null,"deleted_at":null} + """.formatted(objectId, objectKey, hash()).replace("\n", ""); + } + + private static String hash() { + return "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + } +} diff --git a/db-migration/src/test/java/com/idea2strategy/backend/migration/CanonicalMigrationBundleAssemblerTest.java b/db-migration/src/test/java/com/idea2strategy/backend/migration/CanonicalMigrationBundleAssemblerTest.java index eab9512d..7ab7bf1b 100644 --- a/db-migration/src/test/java/com/idea2strategy/backend/migration/CanonicalMigrationBundleAssemblerTest.java +++ b/db-migration/src/test/java/com/idea2strategy/backend/migration/CanonicalMigrationBundleAssemblerTest.java @@ -46,6 +46,8 @@ void assemblesOnlyOwnedCanonicalContributionsInGlobalVersionOrder() throws Excep "V20260826010000__backend_bind_room_invitations_to_accounts.sql", "V20260902000000__pipeline_backtest_object_cleanup_capability.sql", "V20260902000001__pipeline_bind_backtest_cleanup_ownership.sql", + "V20260902000002__backtest_narrow_runtime_attempt_writes.sql", + "V20260902000003__pipeline_narrow_backtest_object_writes.sql", DatabaseAccessPolicy.RUNTIME_GRANTS_FILE), result.orderedFileNames()); assertTrue(Files.readString(result.directory().resolve(DatabaseAccessPolicy.RUNTIME_GRANTS_FILE)) @@ -60,6 +62,12 @@ void assemblesOnlyOwnedCanonicalContributionsInGlobalVersionOrder() throws Excep assertTrue(Files.readString(result.directory().resolve(DatabaseAccessPolicy.RUNTIME_GRANTS_FILE)) .contains("GRANT EXECUTE ON FUNCTION \"storage\".\"reissue_backtest_object_cleanup\"(jsonb, text) " + "TO idea2strategy_backtest")); + assertTrue(Files.readString(result.directory().resolve(DatabaseAccessPolicy.RUNTIME_GRANTS_FILE)) + .contains("GRANT EXECUTE ON FUNCTION \"backtest\".\"claim_run_attempt\"(uuid, text, text, bigint) " + + "TO idea2strategy_backtest")); + assertTrue(Files.readString(result.directory().resolve(DatabaseAccessPolicy.RUNTIME_GRANTS_FILE)) + .contains("GRANT EXECUTE ON FUNCTION \"storage\".\"register_backtest_object\"(jsonb) " + + "TO idea2strategy_backtest")); assertTrue(Files.exists(result.directory().resolve(CanonicalMigrationBundle.MANIFEST_FILE))); assertTrue(Files.exists(result.directory().resolve(CanonicalMigrationBundle.DIGEST_FILE))); } diff --git a/db-migration/src/test/java/com/idea2strategy/backend/migration/DatabaseAccessPolicyTest.java b/db-migration/src/test/java/com/idea2strategy/backend/migration/DatabaseAccessPolicyTest.java index 7f42315e..a983704d 100644 --- a/db-migration/src/test/java/com/idea2strategy/backend/migration/DatabaseAccessPolicyTest.java +++ b/db-migration/src/test/java/com/idea2strategy/backend/migration/DatabaseAccessPolicyTest.java @@ -77,7 +77,7 @@ void limitsPythonRolesToDocumentedReadAndWriteBoundaries() { DatabaseAccessPolicy.Access.UPDATE, "backtest", "runs")); - assertTrue(DatabaseAccessPolicy.allows( + assertFalse(DatabaseAccessPolicy.allows( DatabaseAccessPolicy.ApplicationRole.BACKTEST, DatabaseAccessPolicy.Access.INSERT, "storage", @@ -151,6 +151,59 @@ void limitsPythonRolesToDocumentedReadAndWriteBoundaries() { } } + @Test + void keepsProtectedBacktestAttemptAndStoragePublicationWritesBehindCapabilities() + throws Exception { + String baseline; + try (var input = getClass().getClassLoader().getResourceAsStream("db/migration/V1__initial_schema.sql")) { + baseline = new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + + var sql = DatabaseAccessPolicy.runtimeGrantSql(List.of(baseline)); + + for (var access : List.of(DatabaseAccessPolicy.Access.INSERT, DatabaseAccessPolicy.Access.UPDATE)) { + assertFalse( + DatabaseAccessPolicy.allows( + DatabaseAccessPolicy.ApplicationRole.BACKTEST, + access, + "backtest", + "run_attempts"), + "attempt state must be changed only by a fenced database capability: " + access); + assertFalse( + DatabaseAccessPolicy.allows( + DatabaseAccessPolicy.ApplicationRole.BACKTEST, + access, + "storage", + "objects"), + "publication state must be changed only by a fenced database capability: " + access); + } + assertTrue(sql.contains( + "GRANT SELECT ON TABLE \"backtest\".\"run_attempts\" TO idea2strategy_backtest;")); + assertTrue(sql.contains( + "GRANT SELECT ON TABLE \"storage\".\"objects\" TO idea2strategy_backtest;")); + assertFalse(sql.contains( + "GRANT SELECT, INSERT, UPDATE ON TABLE \"backtest\".\"run_attempts\" " + + "TO idea2strategy_backtest;")); + assertFalse(sql.contains( + "GRANT SELECT, INSERT, UPDATE ON TABLE \"storage\".\"objects\" " + + "TO idea2strategy_backtest;")); + + for (var signature : List.of( + "\"backtest\".\"claim_run_attempt\"(uuid, text, text, bigint)", + "\"backtest\".\"heartbeat_run_attempt\"(uuid, uuid, bigint)", + "\"backtest\".\"close_run_attempt\"(uuid, uuid, text, text, text, boolean)", + "\"backtest\".\"recover_expired_run_attempt\"(uuid, text, text)", + "\"storage\".\"register_backtest_object\"(jsonb)", + "\"storage\".\"transition_backtest_object\"(uuid, text, timestamp with time zone)")) { + assertTrue( + sql.contains("GRANT EXECUTE ON FUNCTION " + signature + " TO idea2strategy_backtest;"), + "the runtime role needs the narrow capability " + signature); + assertTrue( + sql.contains("REVOKE ALL ON FUNCTION " + signature + " FROM PUBLIC;"), + "the narrow capability must not remain executable by PUBLIC: " + signature); + } + } + @Test void rejectsApplicationDdlGrantsInMigrations() { assertThrows( @@ -359,34 +412,27 @@ void grantsTheBacktestRoleTheBotReadsItsExecutorPerforms() { } @Test - void grantsTheBacktestRoleTheStorageObjectPromotionItsRegistrarPerforms() throws Exception { - // INT03 run 9095f2a3 failed five times against a role that held SELECT and INSERT on - // storage.objects and not UPDATE: on the deployed idea2strategy_backtest_runtime, - // has_table_privilege reported select=true, insert=true, update=false. - // - // UPDATE is not a convenience here. StorageObjectRegistrar inserts the row as STAGED and only - // then re-reads the bytes; mark_available and quarantine are both - // `UPDATE storage.objects SET status = ...`, so a run that writes its detail objects cannot - // record that they verified. The row exists, the bytes exist, and the run dies anyway. - for (var access : List.of( + void grantsTheBacktestRoleOnlyCapabilityBasedStorageObjectPublication() throws Exception { + // The worker needs to read object metadata, but staging and promotion now cross narrow, + // attempt-fenced SECURITY DEFINER capabilities. Direct table writes could otherwise forge + // cleanup ownership or publish unverified bytes. + assertTrue(DatabaseAccessPolicy.allows( + DatabaseAccessPolicy.ApplicationRole.BACKTEST, DatabaseAccessPolicy.Access.READ, + "storage", + "objects")); + for (var access : List.of( DatabaseAccessPolicy.Access.INSERT, - DatabaseAccessPolicy.Access.UPDATE)) { - assertTrue( + DatabaseAccessPolicy.Access.UPDATE, + DatabaseAccessPolicy.Access.DELETE)) { + assertFalse( DatabaseAccessPolicy.allows( DatabaseAccessPolicy.ApplicationRole.BACKTEST, access, "storage", "objects"), - "the registrar stages, verifies and promotes its own rows: " + access); + "storage objects must change only through fenced capabilities: " + access); } - // A storage row is the identity of bytes that exist. Verification failure moves it to - // QUARANTINED; nothing in the worker removes one. - assertFalse(DatabaseAccessPolicy.allows( - DatabaseAccessPolicy.ApplicationRole.BACKTEST, - DatabaseAccessPolicy.Access.DELETE, - "storage", - "objects")); // storage.objects is the only table in the schema today, so the rule being table-scoped rather // than schema-scoped is only observable against a name that does not exist yet. Assert it here: @@ -409,16 +455,13 @@ void grantsTheBacktestRoleTheStorageObjectPromotionItsRegistrarPerforms() throws var sql = DatabaseAccessPolicy.runtimeGrantSql(List.of(baseline)); assertTrue( - sql.contains("GRANT SELECT, INSERT, UPDATE ON TABLE \"storage\".\"objects\" " + sql.contains("GRANT SELECT ON TABLE \"storage\".\"objects\" " + "TO idea2strategy_backtest;"), - "the runtime grants must ask for UPDATE, not only SELECT and INSERT"); - assertFalse( - sql.contains("GRANT SELECT, INSERT ON TABLE \"storage\".\"objects\" TO idea2strategy_backtest;"), - "the narrower grant must no longer be generated for the backtest role"); + "the runtime grants retain the metadata read path"); assertFalse( - sql.contains("GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE \"storage\".\"objects\" " + sql.contains("GRANT SELECT, INSERT, UPDATE ON TABLE \"storage\".\"objects\" " + "TO idea2strategy_backtest;"), - "widening must not have handed the worker DELETE"); + "the worker must not receive direct publication writes"); assertTrue( sql.contains("GRANT EXECUTE ON FUNCTION " + "\"storage\".\"prepare_backtest_object_cleanup\"(jsonb) " @@ -435,6 +478,16 @@ void grantsTheBacktestRoleTheStorageObjectPromotionItsRegistrarPerforms() throws assertTrue( sql.contains("REVOKE ALL ON FUNCTION " + "\"storage\".\"reissue_backtest_object_cleanup\"(jsonb, text) FROM PUBLIC;")); + assertTrue( + sql.contains("GRANT EXECUTE ON FUNCTION " + + "\"storage\".\"register_backtest_object\"(jsonb) " + + "TO idea2strategy_backtest;"), + "staging must be exposed only as the narrow registration capability"); + assertTrue( + sql.contains("GRANT EXECUTE ON FUNCTION " + + "\"storage\".\"transition_backtest_object\"(uuid, text, timestamp with time zone) " + + "TO idea2strategy_backtest;"), + "verification must be exposed only as the narrow transition capability"); for (var role : List.of("backend", "batch", "trading", "pipeline")) { assertFalse( sql.contains("GRANT EXECUTE ON FUNCTION " diff --git a/db-migration/src/test/java/com/idea2strategy/backend/migration/MigrationPolicyTest.java b/db-migration/src/test/java/com/idea2strategy/backend/migration/MigrationPolicyTest.java index d4bece42..f3b0f3cd 100644 --- a/db-migration/src/test/java/com/idea2strategy/backend/migration/MigrationPolicyTest.java +++ b/db-migration/src/test/java/com/idea2strategy/backend/migration/MigrationPolicyTest.java @@ -66,7 +66,9 @@ void verifiesTheCheckedInMigrationDirectoryAndBaselineChecksum() throws Exceptio "V20260825000001__pipeline_basic_strategy_feature_catalog.sql", "V20260826010000__backend_bind_room_invitations_to_accounts.sql", "V20260902000000__pipeline_backtest_object_cleanup_capability.sql", - "V20260902000001__pipeline_bind_backtest_cleanup_ownership.sql"), + "V20260902000001__pipeline_bind_backtest_cleanup_ownership.sql", + "V20260902000002__backtest_narrow_runtime_attempt_writes.sql", + "V20260902000003__pipeline_narrow_backtest_object_writes.sql"), plan.orderedFileNames()); } From f41d81dcd71c86a7fc0bb9d0d3ad002b4ec7fb15 Mon Sep 17 00:00:00 2001 From: Ryan Min Date: Wed, 2 Sep 2026 22:23:35 +0900 Subject: [PATCH 11/13] fix: align official manifest policy overlap --- .../OfficialBacktestInputSelector.java | 4 ++-- .../OfficialBacktestInputSelectorTest.java | 20 +++++++++++++++++++ ...ableStrategyReleaseJooqCommandAdapter.java | 11 +++++----- ...tegyReleasePersistenceIntegrationTest.java | 17 +++++++++++++--- 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelector.java b/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelector.java index 7054d8d1..d1891e15 100644 --- a/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelector.java +++ b/modules/backend-application/src/main/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelector.java @@ -71,8 +71,8 @@ private static Selection selectDatasets( List candidates = catalog.datasets().stream() .filter(dataset -> "ADJUSTED".equals(dataset.dataLayer())) .filter(dataset -> policy.marketDataSchemaVersion().equals(dataset.schemaVersion())) - .filter(dataset -> !dataset.periodStart().isBefore(policy.periodStart())) - .filter(dataset -> !dataset.periodEnd().isAfter(policy.periodEnd().plusDays(1))) + .filter(dataset -> dataset.periodEnd().isAfter(policy.periodStart())) + .filter(dataset -> dataset.periodStart().isBefore(policy.periodEnd())) .filter(dataset -> !dataset.availableAt().isAfter(catalog.observedAt())) .filter(dataset -> requirements.resolutions().contains(normalizeResolution(dataset.resolution()))) .toList(); diff --git a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelectorTest.java b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelectorTest.java index 841db50c..f20ee810 100644 --- a/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelectorTest.java +++ b/modules/backend-application/src/test/java/com/idea2strategy/backend/application/strategy/OfficialBacktestInputSelectorTest.java @@ -134,6 +134,26 @@ void selectsTheMinimumOrderedManifestCoverForEveryRequiredResolution() { .containsExactly(bars30mFull, bars1h2024, bars1h2025); } + @Test + void selectsTheFinalAnnualManifestWhenTheLockedPeriodEndsInsideIt() { + UUID bars2024 = UUID.fromString("51000000-0000-4000-8000-000000000001"); + UUID bars2025 = UUID.fromString("51000000-0000-4000-8000-000000000002"); + var catalog = new StrategyReleaseInputCatalog( + List.of(policy("official-v1", "2024-01-01", "2025-07-31", NOW.minusSeconds(60))), + List.of( + dataset(bars2024, "ADJUSTED", "30m", 4, + "2024-01-01", "2025-01-01", NOW.minusSeconds(20)), + dataset(bars2025, "ADJUSTED", "30m", 4, + "2025-01-01", "2026-01-01", NOW.minusSeconds(10))), + NOW); + + var selected = OfficialBacktestInputSelector.select( + plan("30m", "30m"), LocalDate.parse("2024-01-02"), LocalDate.parse("2025-07-30"), catalog); + + assertThat(selected.datasets()).extracting(Dataset::id) + .containsExactly(bars2024, bars2025); + } + @Test void failsClosedAndNamesTheResolutionWhenSegmentedCoverageHasAGap() { UUID first = UUID.fromString("60000000-0000-4000-8000-000000000001"); diff --git a/modules/backend-persistence/src/main/java/com/idea2strategy/backend/persistence/strategy/ImmutableStrategyReleaseJooqCommandAdapter.java b/modules/backend-persistence/src/main/java/com/idea2strategy/backend/persistence/strategy/ImmutableStrategyReleaseJooqCommandAdapter.java index 75b02c46..f80ce50b 100644 --- a/modules/backend-persistence/src/main/java/com/idea2strategy/backend/persistence/strategy/ImmutableStrategyReleaseJooqCommandAdapter.java +++ b/modules/backend-persistence/src/main/java/com/idea2strategy/backend/persistence/strategy/ImmutableStrategyReleaseJooqCommandAdapter.java @@ -426,8 +426,9 @@ static String basicRequestHash( *

The comparison is by calendar date in the policy's own timezone. A legacy * {@code market-bars/1} manifest labels its period with UTC dates while the policy states local * midnight, so comparing instants would reject a pair that does describe the same days; the - * consumer resolves it the same way (backtest-engine #87). Both ends are inclusive of the - * manifest and must lie inside the policy window. + * consumer resolves it the same way (backtest-engine #87). Manifests are immutable storage + * partitions, so a policy may use the intersecting rows of a partition that extends beyond its + * evaluation window; a partition wholly outside the policy cannot describe that replay. */ private OfficialPolicy officialPolicy(String policyDocument) { final com.fasterxml.jackson.databind.JsonNode document; @@ -481,11 +482,11 @@ private void requireCompatibleOfficialInput(OfficialPolicy policy, org.jooq.Reco java.time.LocalDate manifestFirstDay = dataset.get("period_start", java.time.LocalDate.class); java.time.LocalDate manifestLastDay = dataset.get("period_end", java.time.LocalDate.class); - if (manifestFirstDay.isBefore(policy.periodStart()) - || manifestLastDay.isAfter(policy.periodEnd().plusDays(1))) { + if (!manifestLastDay.isAfter(policy.periodStart()) + || !manifestFirstDay.isBefore(policy.periodEnd())) { throw new ImmutableStrategyReleaseRejectedException( "Official backtest dataset period " + manifestFirstDay + ".." + manifestLastDay - + " is not inside the execution policy period " + + " does not overlap the execution policy period " + policy.periodStart() + ".." + policy.periodEnd()); } } diff --git a/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/strategy/ImmutableStrategyReleasePersistenceIntegrationTest.java b/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/strategy/ImmutableStrategyReleasePersistenceIntegrationTest.java index f1ab20e4..c7edfce5 100644 --- a/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/strategy/ImmutableStrategyReleasePersistenceIntegrationTest.java +++ b/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/strategy/ImmutableStrategyReleasePersistenceIntegrationTest.java @@ -431,11 +431,22 @@ private void assertIncompatiblePolicyAndManifestAreRefused( .hasMessageContaining("does not match the execution policy schema"); assertNothingDurable(); - // 2. The manifest period reaches outside the policy window. - setPolicyDocument("2025-06-01T04:00:00Z", "2025-07-01T04:00:00Z", "v1"); + // 2. Annual immutable partitions are allowed to overlap a shorter policy window. The run + // still evaluates only the policy dates, and pinning the full object preserves replay. + setPolicyDocument("2025-01-01T05:00:00Z", "2025-07-31T04:00:00Z", "v1"); + OfficialBacktestRequest annualRequest = OfficialBacktestRequest.forRelease( + release, List.of(DATASET_ID), "backtest-policy-v1"); + new TransactionTemplate(transactionManager).executeWithoutResult(status -> { + assertThat(adapter.saveOnce(release, annualRequest, RUN_ID, 7, HASH_A)).isEqualTo(release); + status.setRollbackOnly(); + }); + assertNothingDurable(); + + // 3. A manifest wholly outside the policy window cannot contribute to its replay. + setPolicyDocument("2024-06-01T04:00:00Z", "2024-07-01T04:00:00Z", "v1"); assertThatThrownBy(() -> adapter.saveOnce(release, request, RUN_ID, 7, HASH_A)) .isInstanceOf(ImmutableStrategyReleaseRejectedException.class) - .hasMessageContaining("is not inside the execution policy period"); + .hasMessageContaining("does not overlap the execution policy period"); assertNothingDurable(); // 3. A RAW manifest would measure the strategy against unadjusted splits and dividends. From 20631a9563954ff70beaadbdd654696e0e7e5cbd Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 3 Sep 2026 14:10:53 +0900 Subject: [PATCH 12/13] fix(backtest): select newest feature materialization --- .../FeatureMaterializationPinResolver.java | 2 +- ...rializationPinResolverIntegrationTest.java | 22 ++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/modules/backend-persistence/src/main/java/com/idea2strategy/backend/persistence/backtest/FeatureMaterializationPinResolver.java b/modules/backend-persistence/src/main/java/com/idea2strategy/backend/persistence/backtest/FeatureMaterializationPinResolver.java index 816356b0..9d546338 100644 --- a/modules/backend-persistence/src/main/java/com/idea2strategy/backend/persistence/backtest/FeatureMaterializationPinResolver.java +++ b/modules/backend-persistence/src/main/java/com/idea2strategy/backend/persistence/backtest/FeatureMaterializationPinResolver.java @@ -121,7 +121,7 @@ private FeaturePin resolveOne( + "where fm.feature_definition_id = ? and fm.instrument_id = ? and fm.status = 'SUCCEEDED' " + "and fm.period_start <= ?::timestamptz and fm.period_end >= ?::timestamptz " + "and fm.available_at <= ?::timestamptz " - + "order by fm.id", + + "order by fm.available_at desc, fm.created_at desc, fm.id desc limit 1", OUTPUT_SCHEMA, asOf, requirement.featureId(), instrumentId, requiredStart, requiredEnd, asOf); if (candidates.size() != 1) { throw new IllegalStateException("Required feature/instrument tuple must resolve to exactly one " diff --git a/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/backtest/FeatureMaterializationPinResolverIntegrationTest.java b/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/backtest/FeatureMaterializationPinResolverIntegrationTest.java index 6afea5c3..7b648f8b 100644 --- a/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/backtest/FeatureMaterializationPinResolverIntegrationTest.java +++ b/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/backtest/FeatureMaterializationPinResolverIntegrationTest.java @@ -135,7 +135,7 @@ void rejectsAPlanWithNoRequiredFeaturesArrayAtAll() { fixture that can seed a second definition end to end belongs in its own change. */ @Test - void rejectsMissingDuplicateAndManifestMismatchBeforeAPinCanBePublished() { + void rejectsMissingAndManifestMismatchBeforeAPinCanBePublished() { jdbc.update("update market_data.feature_materializations set status = 'FAILED', " + "output_dataset_manifest_id = null, result_hash = null, available_at = null where id = ?", MATERIALIZATION); @@ -147,13 +147,6 @@ void rejectsMissingDuplicateAndManifestMismatchBeforeAPinCanBePublished() { jdbc.update("update market_data.feature_materializations set status = 'SUCCEEDED', " + "output_dataset_manifest_id = ?, result_hash = ?, available_at = ? where id = ?", MANIFEST, HASH, AS_OF.minusDays(1), MATERIALIZATION); - seedMaterialization(id(20), id(21), id(22), id(23), id(24), "c".repeat(64)); - assertThatThrownBy(() -> resolver.resolve( - plan(), LocalDate.parse("2024-01-01"), LocalDate.parse("2024-12-31"), AS_OF)) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("exactly one"); - - jdbc.update("delete from market_data.feature_materializations where id = ?", id(20)); jdbc.update("update market_data.dataset_manifests set schema_version = 'unknown.v1' where id = ?", MANIFEST); assertThatThrownBy(() -> resolver.resolve( plan(), LocalDate.parse("2024-01-01"), LocalDate.parse("2024-12-31"), AS_OF)) @@ -161,6 +154,19 @@ void rejectsMissingDuplicateAndManifestMismatchBeforeAPinCanBePublished() { .hasMessageContaining("feature-series.parquet.v1"); } + @Test + void resolvesTheNewestVisibleSucceededRevisionWhenCoverageOverlaps() { + UUID newestMaterialization = id(20); + seedMaterialization(newestMaterialization, id(21), id(22), id(23), id(24), "c".repeat(64)); + jdbc.update("update market_data.feature_materializations set available_at = ?, created_at = ? where id = ?", + AS_OF.minusHours(1), AS_OF.minusHours(2), newestMaterialization); + + assertThat(resolver.resolve( + plan(), LocalDate.parse("2024-01-01"), LocalDate.parse("2024-12-31"), AS_OF)) + .containsExactly(new BacktestRunInputPinWriter.FeaturePin( + newestMaterialization, "sha256:" + HASH)); + } + @Test void rejectsStaleUnavailableAndHashInconsistentPublicationMetadata() { jdbc.update("update market_data.pipeline_runs set output_hash = ? where id = ?", "c".repeat(64), PIPELINE); From eeb4056e6995c9f8384355e9f146d97d0351cf95 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 3 Sep 2026 16:00:04 +0900 Subject: [PATCH 13/13] test(backtest): prove deterministic feature pin selection --- ...erializationPinResolverIntegrationTest.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/backtest/FeatureMaterializationPinResolverIntegrationTest.java b/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/backtest/FeatureMaterializationPinResolverIntegrationTest.java index 7b648f8b..76df2329 100644 --- a/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/backtest/FeatureMaterializationPinResolverIntegrationTest.java +++ b/modules/backend-persistence/src/test/java/com/idea2strategy/backend/persistence/backtest/FeatureMaterializationPinResolverIntegrationTest.java @@ -155,7 +155,7 @@ void rejectsMissingAndManifestMismatchBeforeAPinCanBePublished() { } @Test - void resolvesTheNewestVisibleSucceededRevisionWhenCoverageOverlaps() { + void resolvesExactlyOneNewestVisibleSucceededRevisionWhenCoverageOverlaps() { UUID newestMaterialization = id(20); seedMaterialization(newestMaterialization, id(21), id(22), id(23), id(24), "c".repeat(64)); jdbc.update("update market_data.feature_materializations set available_at = ?, created_at = ? where id = ?", @@ -167,6 +167,22 @@ void resolvesTheNewestVisibleSucceededRevisionWhenCoverageOverlaps() { newestMaterialization, "sha256:" + HASH)); } + @Test + void rejectsAnInvalidNewestRevisionRatherThanFallingBackToAnOlderCandidate() { + UUID newestMaterialization = id(20); + UUID newestManifest = id(22); + seedMaterialization(newestMaterialization, id(21), newestManifest, id(23), id(24), "c".repeat(64)); + jdbc.update("update market_data.feature_materializations set available_at = ?, created_at = ? where id = ?", + AS_OF.minusHours(1), AS_OF.minusHours(2), newestMaterialization); + jdbc.update("update market_data.dataset_manifests set schema_version = 'unknown.v1' where id = ?", + newestManifest); + + assertThatThrownBy(() -> resolver.resolve( + plan(), LocalDate.parse("2024-01-01"), LocalDate.parse("2024-12-31"), AS_OF)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("feature-series.parquet.v1"); + } + @Test void rejectsStaleUnavailableAndHashInconsistentPublicationMetadata() { jdbc.update("update market_data.pipeline_runs set output_hash = ? where id = ?", "c".repeat(64), PIPELINE);