Fix transform index spec

code/go/internal/yamlschema/schema_loader.go

@@ -62,6 +62,9 @@ func (l *yamlReferenceLoader) LoadJSON() (any, error) {
 	if err != nil {
 		return nil, fmt.Errorf("schema unmarshalling failed (path: %s): %w", l.source, err)
 	}
+	if len(schema.Spec) == 0 {
+		return nil, fmt.Errorf("no spec found in schema file (path: %s)", l.source)
+	}
 
 	// fixJSONNumbers ensures that the numbers in the resulting spec are of type `json.Number`, that is
 	// what the gojsonschema library expects. Without this, gojsonschema complains about some integer types

code/go/internal/yamlschema/schema_spec.go

@@ -28,6 +28,7 @@ func (i *itemSchemaSpec) resolve(target semver.Version) (map[string]interface{},
 		// Nothing to do.
 		return i.Spec, nil
 	}
+
 	spec, err := specpatch.ResolvePatch(i.Spec, patchJSON)
 	if err != nil {
 		return nil, fmt.Errorf("failed to apply patch: %w", err)

spec/integration/data_stream/manifest.spec.yml

@@ -247,75 +247,11 @@ spec:
           additionalProperties: false
           properties:
             analysis:
-              type: object
-              additionalProperties: false
-              properties:
-                analyzer:
-                  description: Definition of custom analyzer.
-                  type: object
-                  additionalProperties:
-                    type: object
-                    additionalProperties: false
-                    properties:
-                      type:
-                        description: Type of analyzer
-                        type: string
-                        enum:
-                          - pattern
-                      pattern:
-                        type: string
+              $ref: "#/definitions/index_template_setting_analysis"
             index:
-              type: object
-              additionalProperties: false
-              properties:
-                codec:
-                  description: >
-                    Type of compression to use. The default is to use LZ4, `best_compression` uses DEFLATE,
-                    with higher compression but lower ingestion performance.
-                  type: string
-                  enum:
-                    - default
-                    - best_compression
-                mapping:
-                  type: object
-                  additionalProperties: false
-                  patternProperties:
-                    "^(dimension|nested|total)_fields$":
-                      type: object
-                      additionalProperties: false
-                      properties:
-                        limit:
-                          description: Limit on the number of fields of this kind on this data stream.
-                          type: integer
-                sort:
-                  type: object
-                  additionalProperties: false
-                  properties:
-                    field:
-                      description: Fields used to sort the document in the Lucene segments.
-                      oneOf:
-                        - type: string
-                        - type: array
-                          items:
-                            type: string
-                    order:
-                      description: The sort order to use for each field.
-                      oneOf:
-                        - type: string
-                          enum:
-                            - asc
-                            - desc
-                        - type: array
-                          items:
-                            type: string
-                            enum:
-                              - asc
-                              - desc
+              $ref: "#/definitions/index_template_setting_index"
             number_of_shards:
-              description: Number of primary shards that the data stream should have.
-              type: integer
-              minimum: 1
-              default: 1
+              $ref: "#/definitions/index_template_setting_number_of_shards"
         mappings:
           description: Mappings section of index template
           type: object
@@ -477,7 +413,6 @@ spec:
 
               type: boolean
               default: true
-
         ingest_pipeline:
           description: Elasticsearch ingest pipeline settings
           type: object
@@ -496,6 +431,76 @@ spec:
             hidden:
               description: Makes the data stream hidden
               type: boolean
+    index_template_setting_analysis:
+      type: object
+      additionalProperties: false
+      properties:
+        analyzer:
+          description: Definition of custom analyzer.
+          type: object
+          additionalProperties:
+            type: object
+            additionalProperties: false
+            properties:
+              type:
+                description: Type of analyzer
+                type: string
+                enum:
+                  - pattern
+              pattern:
+                type: string
+    index_template_setting_index:
+      type: object
+      additionalProperties: false
+      properties:
+        codec:
+          description: >
+            Type of compression to use. The default is to use LZ4, `best_compression` uses DEFLATE,
+            with higher compression but lower ingestion performance.
+          type: string
+          enum:
+            - default
+            - best_compression
+        mapping:
+          type: object
+          additionalProperties: false
+          patternProperties:
+            "^(dimension|nested|total)_fields$":
+              type: object
+              additionalProperties: false
+              properties:
+                limit:
+                  description: Limit on the number of fields of this kind on this data stream.
+                  type: integer
+        sort:
+          type: object
+          additionalProperties: false
+          properties:
+            field:
+              description: Fields used to sort the document in the Lucene segments.
+              oneOf:
+                - type: string
+                - type: array
+                  items:
+                    type: string
+            order:
+              description: The sort order to use for each field.
+              oneOf:
+                - type: string
+                  enum:
+                    - asc
+                    - desc
+                - type: array
+                  items:
+                    type: string
+                    enum:
+                      - asc
+                      - desc
+    index_template_setting_number_of_shards:
+      description: Number of primary shards that the data stream should have.
+      type: integer
+      minimum: 1
+      default: 1
   properties:
     dataset:
       description: Name of data set.

spec/integration/elasticsearch/transform/manifest.spec.yml

@@ -1,45 +1,61 @@
-type: object
-additionalProperties: false
-definitions:
-  index_template:
-    $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template"
-properties:
-  destination_index_template:
-    description: Elasticsearch index template for the transform's destination index
-    type: object
-    additionalProperties: false
-    properties:
-      settings:
-        type: object
-        additionalProperties: false
-        properties:
-          index:
-            type: object
-            additionalProperties: false
-            properties:
-              mode:
-                description: Index mode for lookup indices.
-                type: string
-                enum:
-                  - lookup
-              codec:
-                $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/index/properties/codec"
-              mapping:
-                $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/index/properties/mapping"
-              sort:
-                $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/index/properties/sort"
-          number_of_shards:
-            $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/number_of_shards"
-          analysis:
-            $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/settings/properties/analysis"
-      mappings:
-        $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/mappings"
-      ingest_pipeline:
-        $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/ingest_pipeline"
-      data_stream:
-        $ref: "../../data_stream/manifest.spec.yml#/properties/elasticsearch/properties/index_template/properties/data_stream"
-    required: false
-  start:
-    description: Determines if the transform will be started upon installation
-    type: boolean
-    default: true
+spec:
+  type: object
+  additionalProperties: false
+  properties:
+    destination_index_template:
+      description: Elasticsearch index template for the transform's destination index
+      type: object
+      additionalProperties: false
+      properties:
+        settings:
+          type: object
+          additionalProperties: false
+          properties:
+            analysis:
+              $ref: "../../data_stream/manifest.spec.yml#/definitions/index_template_setting_analysis"
+            index:
+              type: object
+              additionalProperties: false
+              properties:
+                codec:
+                  $ref: "../../data_stream/manifest.spec.yml#/definitions/index_template_setting_index/properties/codec"
+                mapping:
+                  $ref: "../../data_stream/manifest.spec.yml#/definitions/index_template_setting_index/properties/mapping"
+                mode:
+                  description: Index mode for lookup indices.
+                  type: string
+                  enum:
+                    - lookup
+                refresh_interval:
+                  description: >
+                    How often to perform a refresh operation, which makes recent changes to the index visible to search.
+                  type: string
+                  example: 1s
+                sort:
+                  $ref: "../../data_stream/manifest.spec.yml#/definitions/index_template_setting_index/properties/sort"
+                number_of_shards:
+                  $ref: "../../data_stream/manifest.spec.yml#/definitions/index_template_setting_number_of_shards"
+                number_of_routing_shards:
+                  description: Number used when splitting shards.
+                  $ref: "../../data_stream/manifest.spec.yml#/definitions/index_template_setting_number_of_shards"
+                hidden:
+                  description: Makes the index hidden.
+                  type: boolean
+                  default: false
+        mappings:
+          $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/mappings"
+        ingest_pipeline:
+          $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/ingest_pipeline"
+        data_stream:
+          $ref: "../../data_stream/manifest.spec.yml#/definitions/elasticsearch_index_template/properties/data_stream"
+    start:
+      description: Determines if the transform will be started upon installation
+      type: boolean
+      default: true
+
+
+versions:
+  - before: 3.4.0
+    patch:
+      - op: remove
+        path: "/properties/destination_index_template/properties/settings/properties/index/properties/mode" # Lookup index mode is not supported in older stacks.