From d21b60d46901b5e80ab1ae015b2520f700a053b5 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 25 Aug 2025 17:44:29 +0200 Subject: [PATCH 1/4] Fix schema --- code/go/internal/yamlschema/schema_spec.go | 1 + .../elasticsearch/transform/manifest.spec.yml | 94 ++++++++++--------- 2 files changed, 50 insertions(+), 45 deletions(-) diff --git a/code/go/internal/yamlschema/schema_spec.go b/code/go/internal/yamlschema/schema_spec.go index 62475531b..2af17d879 100644 --- a/code/go/internal/yamlschema/schema_spec.go +++ b/code/go/internal/yamlschema/schema_spec.go @@ -28,6 +28,7 @@ func (i *itemSchemaSpec) resolve(target semver.Version) (map[string]interface{}, // Nothing to do. return i.Spec, nil } + spec, err := specpatch.ResolvePatch(i.Spec, patchJSON) if err != nil { return nil, fmt.Errorf("failed to apply patch: %w", err) diff --git a/spec/integration/elasticsearch/transform/manifest.spec.yml b/spec/integration/elasticsearch/transform/manifest.spec.yml index 8a3e2f02a..263752a31 100644 --- a/spec/integration/elasticsearch/transform/manifest.spec.yml +++ b/spec/integration/elasticsearch/transform/manifest.spec.yml @@ -1,45 +1,49 @@ -type: object -additionalProperties: false -definitions: - index_template: - $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template" -properties: - destination_index_template: - description: Elasticsearch index template for the transform's destination index - type: object - additionalProperties: false - properties: - settings: - type: object - additionalProperties: false - properties: - index: - type: object - additionalProperties: false - properties: - mode: - description: Index mode for lookup indices. - type: string - enum: - - lookup - codec: - $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/index/properties/codec" - mapping: - $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/index/properties/mapping" - sort: - $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/index/properties/sort" - number_of_shards: - $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/number_of_shards" - analysis: - $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/analysis" - mappings: - $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/mappings" - ingest_pipeline: - $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/ingest_pipeline" - data_stream: - $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/data_stream" - required: false - start: - description: Determines if the transform will be started upon installation - type: boolean - default: true \ No newline at end of file +spec: + type: object + additionalProperties: false + properties: + destination_index_template: + description: Elasticsearch index template for the transform's destination index + type: object + additionalProperties: false + properties: + settings: + type: object + additionalProperties: false + properties: + index: + type: object + additionalProperties: false + properties: + mode: + description: Index mode for lookup indices. + type: string + enum: + - lookup + codec: + $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/settings/properties/index/properties/codec" + mapping: + $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/settings/properties/index/properties/mapping" + sort: + $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/settings/properties/index/properties/sort" + number_of_shards: + $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/settings/properties/number_of_shards" + analysis: + $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/settings/properties/analysis" + mappings: + $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/mappings" + ingest_pipeline: + $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/ingest_pipeline" + data_stream: + $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/data_stream" + start: + description: Determines if the transform will be started upon installation + type: boolean + default: true + + +#versions: +# - before: 3.4.0 +# patch: +# - op: remove +# path: "/properties/destination_index_template/properties/settings/properties/index" # Lookup index mode is not supported in older stacks. From c29c1464f226d0454db4be69df9da9ea0b07848c Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 25 Aug 2025 18:04:05 +0200 Subject: [PATCH 2/4] Fix transform index spec --- .../elasticsearch/transform/manifest.spec.yml | 94 ++++++++++++++++--- 1 file changed, 82 insertions(+), 12 deletions(-) diff --git a/spec/integration/elasticsearch/transform/manifest.spec.yml b/spec/integration/elasticsearch/transform/manifest.spec.yml index 263752a31..52c15a04d 100644 --- a/spec/integration/elasticsearch/transform/manifest.spec.yml +++ b/spec/integration/elasticsearch/transform/manifest.spec.yml @@ -11,6 +11,24 @@ spec: type: object additionalProperties: false properties: + analysis: + type: object + additionalProperties: false + properties: + analyzer: + description: Definition of custom analyzer. + type: object + additionalProperties: + type: object + additionalProperties: false + properties: + type: + description: Type of analyzer + type: string + enum: + - pattern + pattern: + type: string index: type: object additionalProperties: false @@ -21,15 +39,67 @@ spec: enum: - lookup codec: - $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/settings/properties/index/properties/codec" + description: > + Type of compression to use. The default is to use LZ4, `best_compression` uses DEFLATE, + with higher compression but lower ingestion performance. + type: string + enum: + - default + - best_compression + refresh_interval: + description: > + How often to perform a refresh operation, which makes recent changes to the index visible to search. + type: string + example: 1s mapping: - $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/settings/properties/index/properties/mapping" + type: object + additionalProperties: false + patternProperties: + "^(dimension|nested|total)_fields$": + type: object + additionalProperties: false + properties: + limit: + description: Limit on the number of fields of this kind on this data stream. + type: integer sort: - $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/settings/properties/index/properties/sort" - number_of_shards: - $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/settings/properties/number_of_shards" - analysis: - $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/settings/properties/analysis" + type: object + additionalProperties: false + properties: + field: + description: Fields used to sort the document in the Lucene segments. + oneOf: + - type: string + - type: array + items: + type: string + order: + description: The sort order to use for each field. + oneOf: + - type: string + enum: + - asc + - desc + - type: array + items: + type: string + enum: + - asc + - desc + number_of_shards: + description: Number of primary shards that the data stream should have. + type: integer + minimum: 1 + default: 1 + number_of_routing_shards: + description: Number used when splitting shards. + type: integer + minimum: 1 + default: 1 + hidden: + description: Makes the index hidden. + type: boolean + default: false mappings: $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/mappings" ingest_pipeline: @@ -42,8 +112,8 @@ spec: default: true -#versions: -# - before: 3.4.0 -# patch: -# - op: remove -# path: "/properties/destination_index_template/properties/settings/properties/index" # Lookup index mode is not supported in older stacks. +versions: + - before: 3.4.0 + patch: + - op: remove + path: "/properties/destination_index_template/properties/settings/properties/index/properties/mode" # Lookup index mode is not supported in older stacks. From 4521e802c14f29456b66d49d1416461f9b2fbc76 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 25 Aug 2025 18:17:18 +0200 Subject: [PATCH 3/4] Fail if spec is empty for a spec file --- code/go/internal/yamlschema/schema_loader.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/code/go/internal/yamlschema/schema_loader.go b/code/go/internal/yamlschema/schema_loader.go index 624674ce3..ac711a1eb 100644 --- a/code/go/internal/yamlschema/schema_loader.go +++ b/code/go/internal/yamlschema/schema_loader.go @@ -62,6 +62,9 @@ func (l *yamlReferenceLoader) LoadJSON() (any, error) { if err != nil { return nil, fmt.Errorf("schema unmarshalling failed (path: %s): %w", l.source, err) } + if len(schema.Spec) == 0 { + return nil, fmt.Errorf("no spec found in schema file (path: %s)", l.source) + } // fixJSONNumbers ensures that the numbers in the resulting spec are of type `json.Number`, that is // what the gojsonschema library expects. Without this, gojsonschema complains about some integer types From 42985e598149ac23da0b6ecbd79fe43dd2d35303 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 1 Sep 2025 11:29:51 +0200 Subject: [PATCH 4/4] Reuse definitions for index template settings --- .../integration/data_stream/manifest.spec.yml | 141 +++++++++--------- .../elasticsearch/transform/manifest.spec.yml | 74 +-------- 2 files changed, 81 insertions(+), 134 deletions(-) diff --git a/spec/integration/data_stream/manifest.spec.yml b/spec/integration/data_stream/manifest.spec.yml index eedc8466b..7bd4f4b35 100644 --- a/spec/integration/data_stream/manifest.spec.yml +++ b/spec/integration/data_stream/manifest.spec.yml @@ -247,75 +247,11 @@ spec: additionalProperties: false properties: analysis: - type: object - additionalProperties: false - properties: - analyzer: - description: Definition of custom analyzer. - type: object - additionalProperties: - type: object - additionalProperties: false - properties: - type: - description: Type of analyzer - type: string - enum: - - pattern - pattern: - type: string + $ref: "#/definitions/index_template_setting_analysis" index: - type: object - additionalProperties: false - properties: - codec: - description: > - Type of compression to use. The default is to use LZ4, `best_compression` uses DEFLATE, - with higher compression but lower ingestion performance. - type: string - enum: - - default - - best_compression - mapping: - type: object - additionalProperties: false - patternProperties: - "^(dimension|nested|total)_fields$": - type: object - additionalProperties: false - properties: - limit: - description: Limit on the number of fields of this kind on this data stream. - type: integer - sort: - type: object - additionalProperties: false - properties: - field: - description: Fields used to sort the document in the Lucene segments. - oneOf: - - type: string - - type: array - items: - type: string - order: - description: The sort order to use for each field. - oneOf: - - type: string - enum: - - asc - - desc - - type: array - items: - type: string - enum: - - asc - - desc + $ref: "#/definitions/index_template_setting_index" number_of_shards: - description: Number of primary shards that the data stream should have. - type: integer - minimum: 1 - default: 1 + $ref: "#/definitions/index_template_setting_number_of_shards" mappings: description: Mappings section of index template type: object @@ -477,7 +413,6 @@ spec: type: boolean default: true - ingest_pipeline: description: Elasticsearch ingest pipeline settings type: object @@ -496,6 +431,76 @@ spec: hidden: description: Makes the data stream hidden type: boolean + index_template_setting_analysis: + type: object + additionalProperties: false + properties: + analyzer: + description: Definition of custom analyzer. + type: object + additionalProperties: + type: object + additionalProperties: false + properties: + type: + description: Type of analyzer + type: string + enum: + - pattern + pattern: + type: string + index_template_setting_index: + type: object + additionalProperties: false + properties: + codec: + description: > + Type of compression to use. The default is to use LZ4, `best_compression` uses DEFLATE, + with higher compression but lower ingestion performance. + type: string + enum: + - default + - best_compression + mapping: + type: object + additionalProperties: false + patternProperties: + "^(dimension|nested|total)_fields$": + type: object + additionalProperties: false + properties: + limit: + description: Limit on the number of fields of this kind on this data stream. + type: integer + sort: + type: object + additionalProperties: false + properties: + field: + description: Fields used to sort the document in the Lucene segments. + oneOf: + - type: string + - type: array + items: + type: string + order: + description: The sort order to use for each field. + oneOf: + - type: string + enum: + - asc + - desc + - type: array + items: + type: string + enum: + - asc + - desc + index_template_setting_number_of_shards: + description: Number of primary shards that the data stream should have. + type: integer + minimum: 1 + default: 1 properties: dataset: description: Name of data set. diff --git a/spec/integration/elasticsearch/transform/manifest.spec.yml b/spec/integration/elasticsearch/transform/manifest.spec.yml index 52c15a04d..022e489d8 100644 --- a/spec/integration/elasticsearch/transform/manifest.spec.yml +++ b/spec/integration/elasticsearch/transform/manifest.spec.yml @@ -12,90 +12,32 @@ spec: additionalProperties: false properties: analysis: - type: object - additionalProperties: false - properties: - analyzer: - description: Definition of custom analyzer. - type: object - additionalProperties: - type: object - additionalProperties: false - properties: - type: - description: Type of analyzer - type: string - enum: - - pattern - pattern: - type: string + $ref: "../../data_stream/manifest.spec.yml#/definitions/index_template_setting_analysis" index: type: object additionalProperties: false properties: + codec: + $ref: "../../data_stream/manifest.spec.yml#/definitions/index_template_setting_index/properties/codec" + mapping: + $ref: "../../data_stream/manifest.spec.yml#/definitions/index_template_setting_index/properties/mapping" mode: description: Index mode for lookup indices. type: string enum: - lookup - codec: - description: > - Type of compression to use. The default is to use LZ4, `best_compression` uses DEFLATE, - with higher compression but lower ingestion performance. - type: string - enum: - - default - - best_compression refresh_interval: description: > How often to perform a refresh operation, which makes recent changes to the index visible to search. type: string example: 1s - mapping: - type: object - additionalProperties: false - patternProperties: - "^(dimension|nested|total)_fields$": - type: object - additionalProperties: false - properties: - limit: - description: Limit on the number of fields of this kind on this data stream. - type: integer sort: - type: object - additionalProperties: false - properties: - field: - description: Fields used to sort the document in the Lucene segments. - oneOf: - - type: string - - type: array - items: - type: string - order: - description: The sort order to use for each field. - oneOf: - - type: string - enum: - - asc - - desc - - type: array - items: - type: string - enum: - - asc - - desc + $ref: "../../data_stream/manifest.spec.yml#/definitions/index_template_setting_index/properties/sort" number_of_shards: - description: Number of primary shards that the data stream should have. - type: integer - minimum: 1 - default: 1 + $ref: "../../data_stream/manifest.spec.yml#/definitions/index_template_setting_number_of_shards" number_of_routing_shards: description: Number used when splitting shards. - type: integer - minimum: 1 - default: 1 + $ref: "../../data_stream/manifest.spec.yml#/definitions/index_template_setting_number_of_shards" hidden: description: Makes the index hidden. type: boolean