diff --git a/.buildkite/pipeline.trigger.compliance.tests.sh b/.buildkite/pipeline.trigger.compliance.tests.sh index 70076c64c..af83d2e00 100755 --- a/.buildkite/pipeline.trigger.compliance.tests.sh +++ b/.buildkite/pipeline.trigger.compliance.tests.sh @@ -28,8 +28,8 @@ steps: EOF # Generate each test we want to do. -compliance_test 9.1.0-SNAPSHOT 3.4.1 -compliance_test 8.19.0-SNAPSHOT 3.4.1 +compliance_test 9.1.0-SNAPSHOT 3.4.3 +compliance_test 8.19.0-SNAPSHOT 3.4.3 compliance_test 9.0.1 3.3.5 compliance_test 8.14.0 3.1.5 compliance_test 8.9.0 2.7.0 diff --git a/code/go/pkg/validator/validator_test.go b/code/go/pkg/validator/validator_test.go index f27d141ee..9d18d2651 100644 --- a/code/go/pkg/validator/validator_test.go +++ b/code/go/pkg/validator/validator_test.go @@ -35,6 +35,7 @@ func TestValidateFile(t *testing.T) { "good_v3": {}, "good_input": {}, "good_content": {}, + "good_lookup_index": {}, "deploy_custom_agent": {}, "deploy_custom_agent_multi_services": {}, "deploy_docker": {}, diff --git a/compliance/compliance_test.go b/compliance/compliance_test.go index aa4a5c7c2..ebca3000e 100644 --- a/compliance/compliance_test.go +++ b/compliance/compliance_test.go @@ -225,6 +225,12 @@ func indexTemplateIsConfiguredFor(indexTemplateName, option string) error { return nil } return errors.New("synthetic source mode is not enabled") + + case "lookup index mode": + if indexTemplate.Settings.Index.Mode == "lookup" { + return nil + } + return errors.New("lookup mode is not enabled") } return godog.ErrPending diff --git a/compliance/features/esql.feature b/compliance/features/esql.feature new file mode 100644 index 000000000..b576e7217 --- /dev/null +++ b/compliance/features/esql.feature @@ -0,0 +1,8 @@ +Feature: ES|QL + Features related to ES|QL + + @3.4.3 + Scenario: Installer leverages lookup index mode + Given the "good_lookup_index" package is installed + And a policy is created with "good_lookup_index" package and "0.1.4" version + Then index template "logs-good_lookup_index.foo-template" is configured for "lookup index mode" \ No newline at end of file diff --git a/compliance/indextemplate.go b/compliance/indextemplate.go index 3d8f3c902..cafff476b 100644 --- a/compliance/indextemplate.go +++ b/compliance/indextemplate.go @@ -23,6 +23,7 @@ type IndexTemplate struct { type SimulatedIndexTemplate struct { Settings struct { Index struct { + Mode string `json:"mode"` Mapping struct { Source struct { Mode string `json:"mode"` diff --git a/spec/changelog.yml b/spec/changelog.yml index 2610b3860..7a654278f 100644 --- a/spec/changelog.yml +++ b/spec/changelog.yml @@ -13,7 +13,12 @@ - description: Add `duration` variable data type with `min_duration` and `max_duration` validation properties. type: enhancement link: https://github.com/elastic/package-spec/pull/948 -- version: 3.4.2-next +- version: 3.4.3-next + changes: + - description: Add support for lookup indices. + type: enhancement + link: https://github.com/elastic/package-spec/pull/936 +- version: 3.4.2 changes: - description: Add optional docs_structure_enforced parameter to validation.yml. type: enhancement diff --git a/spec/integration/elasticsearch/transform/manifest.spec.yml b/spec/integration/elasticsearch/transform/manifest.spec.yml index bc6ce38b4..8a3e2f02a 100644 --- a/spec/integration/elasticsearch/transform/manifest.spec.yml +++ b/spec/integration/elasticsearch/transform/manifest.spec.yml @@ -1,18 +1,45 @@ -## -## Describes the specification for a transform's manifest.yml file -## -spec: - type: object - additionalProperties: false - definitions: - index_template: - $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template" - properties: - destination_index_template: - description: Elasticsearch index template for the transform's destination index - $ref: "#/definitions/index_template" - required: false - start: - description: Determines if the transform will be started upon installation - type: boolean - default: true \ No newline at end of file +type: object +additionalProperties: false +definitions: + index_template: + $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template" +properties: + destination_index_template: + description: Elasticsearch index template for the transform's destination index + type: object + additionalProperties: false + properties: + settings: + type: object + additionalProperties: false + properties: + index: + type: object + additionalProperties: false + properties: + mode: + description: Index mode for lookup indices. + type: string + enum: + - lookup + codec: + $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/index/properties/codec" + mapping: + $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/index/properties/mapping" + sort: + $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/index/properties/sort" + number_of_shards: + $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/number_of_shards" + analysis: + $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/analysis" + mappings: + $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/mappings" + ingest_pipeline: + $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/ingest_pipeline" + data_stream: + $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/data_stream" + required: false + start: + description: Determines if the transform will be started upon installation + type: boolean + default: true \ No newline at end of file diff --git a/test/packages/good_lookup_index/_dev/build/build.yml b/test/packages/good_lookup_index/_dev/build/build.yml new file mode 100755 index 000000000..97fc8aa10 --- /dev/null +++ b/test/packages/good_lookup_index/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@v8.17.0 diff --git a/test/packages/good_lookup_index/_dev/build/docs/README.md b/test/packages/good_lookup_index/_dev/build/docs/README.md new file mode 100755 index 000000000..98856cfbd --- /dev/null +++ b/test/packages/good_lookup_index/_dev/build/docs/README.md @@ -0,0 +1 @@ +# Good Lookup Index diff --git a/test/packages/good_lookup_index/changelog.yml b/test/packages/good_lookup_index/changelog.yml new file mode 100644 index 000000000..e5fbdc427 --- /dev/null +++ b/test/packages/good_lookup_index/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: 0.1.4 + changes: + - description: "initital" + type: bugfix + link: https://github.com/elastic/integrations/pull/14883 \ No newline at end of file diff --git a/test/packages/good_lookup_index/data_stream/foo/elasticsearch/ingest_pipeline/default.yml b/test/packages/good_lookup_index/data_stream/foo/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 000000000..2f496be16 --- /dev/null +++ b/test/packages/good_lookup_index/data_stream/foo/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,3 @@ +--- +description: "Default empty ingest pipeline for test dataset." +processors: [] \ No newline at end of file diff --git a/test/packages/good_lookup_index/data_stream/foo/fields/base-fields.yml b/test/packages/good_lookup_index/data_stream/foo/fields/base-fields.yml new file mode 100644 index 000000000..7c798f453 --- /dev/null +++ b/test/packages/good_lookup_index/data_stream/foo/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/good_lookup_index/data_stream/foo/manifest.yml b/test/packages/good_lookup_index/data_stream/foo/manifest.yml new file mode 100644 index 000000000..6e81f7127 --- /dev/null +++ b/test/packages/good_lookup_index/data_stream/foo/manifest.yml @@ -0,0 +1,2 @@ +title: "Foo" +type: metrics \ No newline at end of file diff --git a/test/packages/good_lookup_index/docs/README.md b/test/packages/good_lookup_index/docs/README.md new file mode 100644 index 000000000..7e5960073 --- /dev/null +++ b/test/packages/good_lookup_index/docs/README.md @@ -0,0 +1 @@ +# README diff --git a/test/packages/good_lookup_index/elasticsearch/transform/bar/fields/fields.yml b/test/packages/good_lookup_index/elasticsearch/transform/bar/fields/fields.yml new file mode 100644 index 000000000..893a73ab5 --- /dev/null +++ b/test/packages/good_lookup_index/elasticsearch/transform/bar/fields/fields.yml @@ -0,0 +1,3 @@ +- name: "@timestamp" + type: date + description: The timestamp. \ No newline at end of file diff --git a/test/packages/good_lookup_index/elasticsearch/transform/bar/manifest.yml b/test/packages/good_lookup_index/elasticsearch/transform/bar/manifest.yml new file mode 100644 index 000000000..0fa7a472f --- /dev/null +++ b/test/packages/good_lookup_index/elasticsearch/transform/bar/manifest.yml @@ -0,0 +1 @@ +start: true \ No newline at end of file diff --git a/test/packages/good_lookup_index/elasticsearch/transform/bar/transform.yml b/test/packages/good_lookup_index/elasticsearch/transform/bar/transform.yml new file mode 100644 index 000000000..c0d4e4cdc --- /dev/null +++ b/test/packages/good_lookup_index/elasticsearch/transform/bar/transform.yml @@ -0,0 +1,28 @@ +description: "Test transform that always runs and creates an index." +source: + index: + - ".ds-.logs-elasticsearch.deprecation-default-*" +dest: + index: test-bar-dest-index +frequency: 1m +sync: + time: + field: "@timestamp" + delay: 0s +pivot: + group_by: + "@timestamp": + date_histogram: + field: "@timestamp" + calendar_interval: 1d + aggregations: + doc_count: + value_count: + field: "_id" +settings: + deduce_mappings: false + unattended: true +_meta: + managed: true + run_as_kibana_system: false + fleet_transform_version: 0.1.0 \ No newline at end of file diff --git a/test/packages/good_lookup_index/elasticsearch/transform/foo/fields/fields.yml b/test/packages/good_lookup_index/elasticsearch/transform/foo/fields/fields.yml new file mode 100644 index 000000000..893a73ab5 --- /dev/null +++ b/test/packages/good_lookup_index/elasticsearch/transform/foo/fields/fields.yml @@ -0,0 +1,3 @@ +- name: "@timestamp" + type: date + description: The timestamp. \ No newline at end of file diff --git a/test/packages/good_lookup_index/elasticsearch/transform/foo/manifest.yml b/test/packages/good_lookup_index/elasticsearch/transform/foo/manifest.yml new file mode 100644 index 000000000..83c8a54fd --- /dev/null +++ b/test/packages/good_lookup_index/elasticsearch/transform/foo/manifest.yml @@ -0,0 +1,5 @@ +start: true +destination_index_template: + settings: + index: + mode: lookup diff --git a/test/packages/good_lookup_index/elasticsearch/transform/foo/transform.yml b/test/packages/good_lookup_index/elasticsearch/transform/foo/transform.yml new file mode 100644 index 000000000..21d80a9b6 --- /dev/null +++ b/test/packages/good_lookup_index/elasticsearch/transform/foo/transform.yml @@ -0,0 +1,28 @@ +description: "Test transform that always runs and creates an index." +source: + index: + - ".ds-.logs-elasticsearch.deprecation-default-*" +dest: + index: test-foo-dest-index +frequency: 1m +sync: + time: + field: "@timestamp" + delay: 0s +pivot: + group_by: + "@timestamp": + date_histogram: + field: "@timestamp" + calendar_interval: 1d + aggregations: + doc_count: + value_count: + field: "_id" +settings: + deduce_mappings: false + unattended: true +_meta: + managed: true + run_as_kibana_system: false + fleet_transform_version: 0.1.0 \ No newline at end of file diff --git a/test/packages/good_lookup_index/img/chargeback.png b/test/packages/good_lookup_index/img/chargeback.png new file mode 100644 index 000000000..971f878e9 Binary files /dev/null and b/test/packages/good_lookup_index/img/chargeback.png differ diff --git a/test/packages/good_lookup_index/img/elastic-logo.png b/test/packages/good_lookup_index/img/elastic-logo.png new file mode 100644 index 000000000..d7a56a3ec Binary files /dev/null and b/test/packages/good_lookup_index/img/elastic-logo.png differ diff --git a/test/packages/good_lookup_index/manifest.yml b/test/packages/good_lookup_index/manifest.yml new file mode 100644 index 000000000..834c5813e --- /dev/null +++ b/test/packages/good_lookup_index/manifest.yml @@ -0,0 +1,18 @@ + +format_version: 3.4.3 +name: good_lookup_index +title: "good_lookup_index" +version: 0.1.4 +description: "This package tests good_lookup_index" +type: integration +categories: + - cloud + - custom +conditions: + kibana: + version: "8.18.0" + elastic: + subscription: "basic" +owner: + github: elastic/customer-architects + type: elastic diff --git a/test/packages/good_lookup_index/validation.yml b/test/packages/good_lookup_index/validation.yml new file mode 100644 index 000000000..bcc8f74ac --- /dev/null +++ b/test/packages/good_lookup_index/validation.yml @@ -0,0 +1,3 @@ +errors: + exclude_checks: + - SVR00002