Add 'lookup' as valid index mode for ES|QL lookup joins

.buildkite/pipeline.trigger.compliance.tests.sh

@@ -28,8 +28,8 @@ steps:
 EOF
 
 # Generate each test we want to do.
-compliance_test 9.1.0-SNAPSHOT 3.4.1
-compliance_test 8.19.0-SNAPSHOT 3.4.1
+compliance_test 9.1.0-SNAPSHOT 3.4.3
+compliance_test 8.19.0-SNAPSHOT 3.4.3
 compliance_test 9.0.1 3.3.5
 compliance_test 8.14.0 3.1.5
 compliance_test 8.9.0 2.7.0

code/go/pkg/validator/validator_test.go

@@ -35,6 +35,7 @@ func TestValidateFile(t *testing.T) {
 		"good_v3":                            {},
 		"good_input":                         {},
 		"good_content":                       {},
+		"good_lookup_index":                  {},
 		"deploy_custom_agent":                {},
 		"deploy_custom_agent_multi_services": {},
 		"deploy_docker":                      {},

compliance/compliance_test.go

@@ -225,6 +225,12 @@ func indexTemplateIsConfiguredFor(indexTemplateName, option string) error {
 			return nil
 		}
 		return errors.New("synthetic source mode is not enabled")
+
+	case "lookup index mode":
+		if indexTemplate.Settings.Index.Mode == "lookup" {
+			return nil
+		}
+		return errors.New("lookup mode is not enabled")
 	}
 
 	return godog.ErrPending

compliance/features/esql.feature

@@ -0,0 +1,8 @@
+Feature: ES|QL
+  Features related to ES|QL
+
+  @3.4.3
+  Scenario: Installer leverages lookup index mode
+   Given the "good_lookup_index" package is installed
+     And a policy is created with "good_lookup_index" package and "0.1.4" version
+    Then index template "logs-good_lookup_index.foo-template" is configured for "lookup index mode"

compliance/indextemplate.go

@@ -23,6 +23,7 @@ type IndexTemplate struct {
 type SimulatedIndexTemplate struct {
 	Settings struct {
 		Index struct {
+			Mode    string `json:"mode"`
 			Mapping struct {
 				Source struct {
 					Mode string `json:"mode"`

spec/changelog.yml

@@ -13,7 +13,12 @@
     - description: Add `duration` variable data type with `min_duration` and `max_duration` validation properties.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/948
-- version: 3.4.2-next
+- version: 3.4.3-next
+  changes:
+    - description: Add support for lookup indices.
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/936
+- version: 3.4.2
   changes:
     - description: Add optional docs_structure_enforced parameter to validation.yml.
       type: enhancement

spec/integration/elasticsearch/transform/manifest.spec.yml

@@ -1,18 +1,45 @@
-##
-## Describes the specification for a transform's manifest.yml file
-##
-spec:
-  type: object
-  additionalProperties: false
-  definitions:
-    index_template:
-      $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template"
-  properties:
-    destination_index_template:
-      description: Elasticsearch index template for the transform's destination index
-      $ref: "#/definitions/index_template"
-      required: false
-    start:
-      description: Determines if the transform will be started upon installation
-      type: boolean
-      default: true
+type: object
+additionalProperties: false
+definitions:
+  index_template:
+    $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template"
+properties:
+  destination_index_template:
+    description: Elasticsearch index template for the transform's destination index
+    type: object
+    additionalProperties: false
+    properties:
+      settings:
+        type: object
+        additionalProperties: false
+        properties:
+          index:
+            type: object
+            additionalProperties: false
+            properties:
+              mode:
+                description: Index mode for lookup indices.
+                type: string
+                enum:
+                  - lookup
+              codec:
+                $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/index/properties/codec"
+              mapping:
+                $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/index/properties/mapping"
+              sort:
+                $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/index/properties/sort"
+          number_of_shards:
+            $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/number_of_shards"
+          analysis:
+            $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/analysis"
+      mappings:
+        $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/mappings"
+      ingest_pipeline:
+        $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/ingest_pipeline"
+      data_stream:
+        $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/data_stream"
+    required: false
+  start:
+    description: Determines if the transform will be started upon installation
+    type: boolean
+    default: true

test/packages/good_lookup_index/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@v8.17.0

test/packages/good_lookup_index/_dev/build/docs/README.md

@@ -0,0 +1 @@
+# Good Lookup Index

test/packages/good_lookup_index/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: 0.1.4
+  changes:
+    - description: "initital"
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/14883

test/packages/good_lookup_index/data_stream/foo/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,3 @@
+---
+description: "Default empty ingest pipeline for test dataset."
+processors: []

test/packages/good_lookup_index/data_stream/foo/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/good_lookup_index/data_stream/foo/manifest.yml

@@ -0,0 +1,2 @@
+title: "Foo"
+type: metrics

test/packages/good_lookup_index/docs/README.md

@@ -0,0 +1 @@
+# README

test/packages/good_lookup_index/elasticsearch/transform/bar/fields/fields.yml

@@ -0,0 +1,3 @@
+- name: "@timestamp"
+  type: date
+  description: The timestamp.

test/packages/good_lookup_index/elasticsearch/transform/bar/manifest.yml

@@ -0,0 +1 @@
+start: true

test/packages/good_lookup_index/elasticsearch/transform/bar/transform.yml

@@ -0,0 +1,28 @@
+description: "Test transform that always runs and creates an index."
+source:
+  index:
+    - ".ds-.logs-elasticsearch.deprecation-default-*"
+dest:
+  index: test-bar-dest-index
+frequency: 1m
+sync:
+  time:
+    field: "@timestamp"
+    delay: 0s
+pivot:
+  group_by:
+    "@timestamp":
+      date_histogram:
+        field: "@timestamp"
+        calendar_interval: 1d
+  aggregations:
+    doc_count:
+      value_count:
+        field: "_id"
+settings:
+  deduce_mappings: false
+  unattended: true
+_meta:
+  managed: true
+  run_as_kibana_system: false
+  fleet_transform_version: 0.1.0

test/packages/good_lookup_index/elasticsearch/transform/foo/fields/fields.yml

@@ -0,0 +1,3 @@
+- name: "@timestamp"
+  type: date
+  description: The timestamp.

test/packages/good_lookup_index/elasticsearch/transform/foo/manifest.yml

@@ -0,0 +1,5 @@
+start: true
+destination_index_template:
+  settings:
+    index:
+      mode: lookup

test/packages/good_lookup_index/elasticsearch/transform/foo/transform.yml

@@ -0,0 +1,28 @@
+description: "Test transform that always runs and creates an index."
+source:
+  index:
+    - ".ds-.logs-elasticsearch.deprecation-default-*"
+dest:
+  index: test-foo-dest-index
+frequency: 1m
+sync:
+  time:
+    field: "@timestamp"
+    delay: 0s
+pivot:
+  group_by:
+    "@timestamp":
+      date_histogram:
+        field: "@timestamp"
+        calendar_interval: 1d
+  aggregations:
+    doc_count:
+      value_count:
+        field: "_id"
+settings:
+  deduce_mappings: false
+  unattended: true
+_meta:
+  managed: true
+  run_as_kibana_system: false
+  fleet_transform_version: 0.1.0

test/packages/good_lookup_index/manifest.yml

@@ -0,0 +1,18 @@
+
+format_version: 3.4.3
+name: good_lookup_index
+title: "good_lookup_index"
+version: 0.1.4
+description: "This package tests good_lookup_index"
+type: integration
+categories:
+  - cloud
+  - custom
+conditions:
+  kibana:
+    version: "8.18.0"
+  elastic:
+    subscription: "basic"
+owner:
+  github: elastic/customer-architects
+  type: elastic

test/packages/good_lookup_index/validation.yml

@@ -0,0 +1,3 @@
+errors:
+  exclude_checks:
+    - SVR00002