feat(stovepipe): add request and request-log entity stores

Makefile

@@ -379,7 +379,7 @@ local-stovepipe-gateway-start: build-stovepipe-gateway-linux ## Start Stovepipe
 
 mocks: ## Generate mock files using mockgen
 	@echo "Generating mocks..."
-	@$(BAZEL) run @rules_go//go -- generate ./submitqueue/extension/storage/... ./submitqueue/extension/buildrunner/... ./submitqueue/extension/changeprovider/... ./platform/extension/counter/... ./platform/extension/messagequeue/... ./submitqueue/extension/queueconfig/... ./submitqueue/extension/mergechecker/... ./submitqueue/extension/pusher/... ./submitqueue/extension/scorer/... ./submitqueue/extension/conflict/... ./platform/consumer/...
+	@$(BAZEL) run @rules_go//go -- generate ./submitqueue/extension/storage/... ./submitqueue/extension/buildrunner/... ./submitqueue/extension/changeprovider/... ./platform/extension/counter/... ./platform/extension/messagequeue/... ./submitqueue/extension/queueconfig/... ./submitqueue/extension/mergechecker/... ./submitqueue/extension/pusher/... ./submitqueue/extension/scorer/... ./submitqueue/extension/conflict/... ./platform/consumer/... ./stovepipe/extension/storage/...
 	@echo "Mocks generated successfully!"
 
 proto: ## Generate protobuf files from .proto definitions

stovepipe/entity/BUILD.bazel

@@ -2,15 +2,23 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "entity",
-    srcs = ["ingest_request.go"],
+    srcs = [
+        "ingest_request.go",
+        "request.go",
+        "request_log.go",
+    ],
     importpath = "github.com/uber/submitqueue/stovepipe/entity",
     visibility = ["//visibility:public"],
     deps = ["//platform/base/change"],
 )
 
 go_test(
     name = "entity_test",
-    srcs = ["ingest_request_test.go"],
+    srcs = [
+        "ingest_request_test.go",
+        "request_log_test.go",
+        "request_test.go",
+    ],
     embed = [":entity"],
     deps = [
         "//platform/base/change",

stovepipe/entity/request.go

@@ -0,0 +1,119 @@
+// Copyright (c) 2025 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package entity
+
+import (
+	"encoding/json"
+
+	"github.com/uber/submitqueue/platform/base/change"
+)
+
+// RequestState defines the possible states of a Stovepipe trunk-validation request. They are internal
+// and used to implement a state machine. A separate RequestStatus type tracks the customer-friendly
+// status of a request. Stovepipe validates commits after they land, so the terminal states describe a
+// commit's health (succeeded / failed) rather than a merge outcome.
+type RequestState string
+
+const (
+	// RequestStateUnknown is the unreachable sentinel state. It is set by default when the structure is
+	// initialized and should never be seen in the system.
+	RequestStateUnknown RequestState = ""
+	// RequestStateStarted is the initial state of a request. The commit has been recorded by the
+	// orchestrator and is entering the pipeline, but no validation has happened yet.
+	RequestStateStarted RequestState = "started"
+	// RequestStateValidated indicates that the commit metadata needed for ordering and batching has been
+	// resolved successfully.
+	RequestStateValidated RequestState = "validated"
+	// RequestStateBatched indicates that the request has been enrolled in a validation batch (a contiguous
+	// range of commits since the last known green).
+	RequestStateBatched RequestState = "batched"
+	// RequestStateBuilding indicates that the batch containing the request is being built and tested.
+	RequestStateBuilding RequestState = "building"
+	// RequestStateSucceeded is the terminal state of a request whose commit validated green. This is a
+	// final state.
+	RequestStateSucceeded RequestState = "succeeded"
+	// RequestStateFailed is the terminal state of a request whose commit was found to break a target. This
+	// is a final state.
+	RequestStateFailed RequestState = "failed"
+	// RequestStateError is the terminal state of a request that encountered an infrastructure error and
+	// could not be validated. This is a final state.
+	RequestStateError RequestState = "error"
+)
+
+// IsRequestStateTerminal returns true if the state represents a final, irreversible state (succeeded,
+// failed, or error). Forward-progress controllers use this to short-circuit work for requests that have
+// already reached a terminal outcome.
+func IsRequestStateTerminal(s RequestState) bool {
+	return s == RequestStateSucceeded || s == RequestStateFailed || s == RequestStateError
+}
+
+// Request defines a Stovepipe request to validate a set of trunk commits after they have landed on the
+// target branch. The immutable fields are fixed at creation; the mutable fields advance as the request
+// moves through the pipeline.
+type Request struct {
+	// ****************
+	// Immutable fields, fixed at request entity creation
+	// ****************
+
+	// ID is the globally unique identifier for the request (the "spid"). Format: "<queue>/<counter_value>".
+	ID string `json:"id"`
+	// Queue is the name of the queue processing the request. Queue name is defined in the configuration and
+	// should be unique within the system.
+	Queue string `json:"queue"`
+	// Change is the set of trunk commits to validate, identified by URI. The scheme names the VCS; the rest
+	// is provider-specific (e.g. git://remote/repo/ref/commit_sha).
+	Change change.Change `json:"change"`
+
+	// ****************
+	// Following fields could be changed throughout the lifecycle of the request
+	// ****************
+
+	// State is the current state of the request.
+	State RequestState `json:"state"`
+	// Version is the version of the object. It is used for optimistic locking.
+	// Versioning starts at 1 and is incremented for each change to the object.
+	Version int32 `json:"version"`
+}
+
+// ToBytes serializes the Request to JSON bytes for queue message payload.
+func (r Request) ToBytes() ([]byte, error) {
+	return json.Marshal(r)
+}
+
+// RequestFromBytes deserializes a Request from JSON bytes.
+func RequestFromBytes(data []byte) (Request, error) {
+	var req Request
+	err := json.Unmarshal(data, &req)
+	return req, err
+}
+
+// RequestID is a lightweight entity for publishing and consuming just the request identifier via the
+// queue. Internal pipeline hops carry the ID and reload the full request from storage.
+type RequestID struct {
+	// ID is the globally unique identifier for the request.
+	ID string `json:"id"`
+}
+
+// ToBytes serializes the RequestID to JSON bytes for queue message payload.
+func (r RequestID) ToBytes() ([]byte, error) {
+	return json.Marshal(r)
+}
+
+// RequestIDFromBytes deserializes a RequestID from JSON bytes.
+func RequestIDFromBytes(data []byte) (RequestID, error) {
+	var rid RequestID
+	err := json.Unmarshal(data, &rid)
+	return rid, err
+}

stovepipe/entity/request_log.go

@@ -0,0 +1,141 @@
+// Copyright (c) 2025 Uber Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package entity
+
+import (
+	"encoding/json"
+	"time"
+)
+
+// RequestStatus defines the possible customer-friendly status of a request. Status is display-friendly
+// and can be shown to callers; it is different from the internal RequestState used to implement the state
+// machine. Request statuses can generally be added freely by the system without breaking the state machine.
+// Some statuses correspond to a request state, in which case they should be supplemented with the request
+// state version for reconciliation. Other statuses are purely informational and can be added freely. Every
+// status may be accompanied by a last error message and free-form metadata in the request log; those are
+// used for display or debugging purposes only.
+type RequestStatus string
+
+const (
+	// RequestStatusUnknown is the unknown sentinel status. It is set by default when the structure is
+	// initialized and should never be seen in the system.
+	RequestStatusUnknown RequestStatus = ""
+
+	// RequestStatusAccepted indicates that the request has been accepted by the system. The gateway sets
+	// this status when the ingest request is received and persisted to the logging database.
+	RequestStatusAccepted RequestStatus = "accepted"
+
+	// RequestStatusStarted is the initial status of a request. It corresponds to the RequestStateStarted
+	// state and is typically set by the orchestrator when the commit is recorded in the operating database.
+	RequestStatusStarted RequestStatus = "started"
+
+	// RequestStatusValidating indicates that the request's commit metadata is currently being resolved for
+	// ordering and batching.
+	RequestStatusValidating RequestStatus = "validating"
+
+	// RequestStatusValidated indicates that the request has been validated successfully. It corresponds to
+	// the RequestStateValidated state.
+	RequestStatusValidated RequestStatus = "validated"
+
+	// RequestStatusBatching indicates that the request is waiting to be included in a validation batch.
+	RequestStatusBatching RequestStatus = "batching"
+
+	// RequestStatusBatched indicates that the request has been included in a validation batch. It
+	// corresponds to the RequestStateBatched state.
+	RequestStatusBatched RequestStatus = "batched"
+
+	// RequestStatusBuilding indicates that the batch containing the request is being built and tested. It
+	// corresponds to the RequestStateBuilding state.
+	RequestStatusBuilding RequestStatus = "building"
+
+	// RequestStatusBuilt indicates that the batch containing the request has finished building and can move
+	// to the next phase.
+	RequestStatusBuilt RequestStatus = "built"
+
+	// RequestStatusSucceeded indicates that the request's commit validated green. It corresponds to the
+	// RequestStateSucceeded state.
+	RequestStatusSucceeded RequestStatus = "succeeded"
+
+	// RequestStatusFailed indicates that the request's commit was found to break a target. It corresponds
+	// to the RequestStateFailed state.
+	RequestStatusFailed RequestStatus = "failed"
+
+	// RequestStatusError indicates that the request encountered an infrastructure error and could not be
+	// validated. It corresponds to the RequestStateError state.
+	RequestStatusError RequestStatus = "error"
+)
+
+// RequestLog is an append-only record that captures a point-in-time snapshot of a request's status for
+// reconciliation purposes. It is stored in a separate database from the request store to support eventual
+// consistency reconciliation.
+type RequestLog struct {
+	// RequestID is the ID of the request this log entry belongs to. References entity.Request.ID.
+	RequestID string `json:"request_id"`
+	// TimestampMs is the time this log entry was created, in milliseconds since Unix epoch.
+	TimestampMs int64 `json:"timestamp_ms"`
+	// Status is the request status at the time this log entry was created. It may contain request states
+	// from the state machine and also display-friendly intermediate statuses.
+	Status RequestStatus `json:"status"`
+	// RequestVersion is the version of the request at the time this log entry was created.
+	// Zero if the version is not available.
+	RequestVersion int32 `json:"request_version"`
+	// LastError is the last error message associated with the status at the time of this log entry.
+	// Empty string if no error.
+	LastError string `json:"last_error"`
+	// Metadata is a set of key-value pairs providing additional context for this log entry.
+	// Empty map if no metadata.
+	Metadata map[string]string `json:"metadata"`
+}
+
+// NewRequestLog creates a new RequestLog with the given fields.
+// TimestampMs is set to the current time. If metadata is nil, it will be initialized as an empty map.
+// requestVersion is the version of the request entity; it should only be set when reporting a request
+// state as a status, otherwise it should be 0.
+// lastError is the last error message associated with the status at the time of this log entry, empty
+// string if no error.
+// metadata is a set of key-value pairs providing additional context for this log entry. It is not
+// constrained to any specific format or schema and is used for display or debugging purposes.
+func NewRequestLog(requestID string, status RequestStatus, requestVersion int32, lastError string, metadata map[string]string) RequestLog {
+	if metadata == nil {
+		metadata = make(map[string]string)
+	}
+	return RequestLog{
+		RequestID:      requestID,
+		TimestampMs:    time.Now().UnixMilli(),
+		Status:         status,
+		RequestVersion: requestVersion,
+		LastError:      lastError,
+		Metadata:       metadata,
+	}
+}
+
+// ToBytes serializes the RequestLog to JSON bytes for queue message payload.
+func (r RequestLog) ToBytes() ([]byte, error) {
+	return json.Marshal(r)
+}
+
+// RequestLogFromBytes deserializes a RequestLog from JSON bytes.
+// If metadata is absent from the JSON, it will be initialized as an empty map.
+func RequestLogFromBytes(data []byte) (RequestLog, error) {
+	var log RequestLog
+	err := json.Unmarshal(data, &log)
+	if err != nil {
+		return log, err
+	}
+	if log.Metadata == nil {
+		log.Metadata = make(map[string]string)
+	}
+	return log, nil
+}