Add source and build validation modes

.gitignore

@@ -6,3 +6,5 @@ fuzz
 /build/
 .vscode/
 .DS_Store
+
+.scratch/

code/go/internal/validator/folder_item_spec.go

@@ -5,10 +5,12 @@
 package validator
 
 import (
+	"errors"
 	"fmt"
 	"io/fs"
 	"regexp"
 
+	"github.com/elastic/package-spec/v3/code/go/internal/linkedfiles"
 	"github.com/elastic/package-spec/v3/code/go/internal/spectypes"
 	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
 )
@@ -40,6 +42,11 @@ func matchingFileExists(spec spectypes.ItemSpec, files []fs.DirEntry) (bool, err
 func validateFile(spec spectypes.ItemSpec, fsys fs.FS, itemPath string) specerrors.ValidationErrors {
 	err := validateMaxSize(fsys, itemPath, spec)
 	if err != nil {
+		if errors.Is(err, linkedfiles.ErrUnsupportedLinkFile) {
+			// .link files in build mode: skip syntactic checks here; the semantic
+			// ValidateNoLinkFiles rule emits a clean validation error for them.
+			return nil
+		}
 		return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)}
 	}
 	if mediaType := spec.ContentMediaType(); mediaType != nil {

code/go/internal/validator/modes/modes.go

@@ -0,0 +1,24 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package modes
+
+// Mode represents the validation mode used when validating a package.
+type Mode string
+
+// Validation modes. The public API re-exports these as validator.Mode* constants.
+const (
+	Legacy Mode = "legacy"
+	Source Mode = "source"
+	Build  Mode = "build"
+)
+
+// Valid reports whether m is a recognised validation mode.
+func (m Mode) Valid() bool {
+	switch m {
+	case Legacy, Source, Build:
+		return true
+	}
+	return false
+}

code/go/internal/validator/semantic/types.go

@@ -352,9 +352,11 @@ func listDataStreams(fsys fspath.FS) ([]string, error) {
 		return nil, fmt.Errorf("can't list data streams directory: %w", err)
 	}
 
-	list := make([]string, len(dataStreams))
-	for i, dataStream := range dataStreams {
-		list[i] = dataStream.Name()
+	var list []string
+	for _, dataStream := range dataStreams {
+		if dataStream.IsDir() {
+			list = append(list, dataStream.Name())
+		}
 	}
 	return list, nil
 }

code/go/internal/validator/semantic/validate_datastream_package_categories.go

@@ -105,7 +105,7 @@ func ValidateDatastreamPackageCategories(fsys fspath.FS) specerrors.ValidationEr
 			specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)}
 	}
 
-	if pkgType != packageTypeIntegration {
+	if pkgType != integrationPackageType {
 		return nil
 	}
 

code/go/internal/validator/semantic/validate_integration_inputs_deprecated.go

@@ -47,7 +47,7 @@ func ValidateIntegrationInputsDeprecation(fsys fspath.FS) specerrors.ValidationE
 			specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), err)}
 	}
 	// skip if not an integration package
-	if m.Type != packageTypeIntegration {
+	if m.Type != integrationPackageType {
 		return nil
 	}
 

code/go/internal/validator/semantic/validate_integration_policy_template_path.go

@@ -20,11 +20,11 @@ import (
 
 const (
 	defaultStreamTemplatePath = "stream.yml.hbs"
-	packageTypeIntegration    = "integration"
 )
 
 type policyTemplateInput struct {
 	Type          string   `yaml:"type"`
+	Package       string   `yaml:"package"`
 	TemplatePath  string   `yaml:"template_path"`
 	TemplatePaths []string `yaml:"template_paths"`
 }
@@ -41,6 +41,7 @@ type integrationPackageManifest struct { // package manifest
 
 type stream struct {
 	Input         string   `yaml:"input"`
+	Package       string   `yaml:"package"`
 	TemplatePath  string   `yaml:"template_path"`
 	TemplatePaths []string `yaml:"template_paths"`
 }
@@ -78,7 +79,7 @@ func ValidateIntegrationPolicyTemplates(fsys fspath.FS) specerrors.ValidationErr
 			specerrors.NewStructuredErrorf("file \"%s\" is invalid: %w", fsys.Path(manifestPath), errFailedToParseManifest)}
 	}
 
-	if manifest.Type != packageTypeIntegration {
+	if manifest.Type != integrationPackageType {
 		return nil
 	}
 
@@ -110,6 +111,14 @@ func ValidateIntegrationPolicyTemplates(fsys fspath.FS) specerrors.ValidationErr
 // under agent/input when template_paths or template_path is set (Fleet: template_paths first).
 func validateIntegrationPolicyTemplateInputs(fsys fspath.FS, policyTemplate integrationPolicyTemplate) error {
 	for _, input := range policyTemplate.Inputs {
+		// Composable inputs reference an input package via 'package:'. When no
+		// explicit template_path/template_paths is set, all templates come from
+		// the dependency and are only present after build. Skip those.
+		// If the composable input defines its own template_path or template_paths
+		// (overlay templates that live in the source package), those are validated.
+		if input.Package != "" && input.TemplatePath == "" && len(input.TemplatePaths) == 0 {
+			continue
+		}
 		if len(input.TemplatePaths) > 0 {
 			for _, tp := range input.TemplatePaths {
 				if err := validateAgentInputTemplatePath(fsys, tp); err != nil {
@@ -141,6 +150,17 @@ func validateAllDataStreamStreamTemplates(fsys fspath.FS, dsMap map[string]dataS
 		dsManifestPath := path.Join(dsDir, "manifest.yml")
 		manifest := dsMap[dsDir]
 		for _, s := range manifest.Streams {
+			// Composable streams reference an input package via 'package:'. When
+			// no explicit template_path/template_paths is set on the stream, all
+			// templates come from the dependency and are only present after build.
+			// Skip those — ValidateStreamInputMaterialized enforces that 'package:'
+			// is replaced by 'input:' in build mode.
+			// However, if the composable stream defines its own template_path or
+			// template_paths, those files must exist in the source package and are
+			// validated here.
+			if s.Package != "" && s.TemplatePath == "" && len(s.TemplatePaths) == 0 {
+				continue
+			}
 			if err := validateSingleDataStreamStreamTemplates(fsys, dsDir, s); err != nil {
 				errs = append(errs, specerrors.NewStructuredErrorf(
 					"file \"%s\" is invalid: data stream \"%s\" stream input %q: %w",

code/go/internal/validator/semantic/validate_integration_policy_template_path_test.go

@@ -254,6 +254,65 @@ streams:
 		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
 		require.Empty(t, errs)
 	})
+
+	t.Run("composable stream with no explicit templates skips validation", func(t *testing.T) {
+		d := t.TempDir()
+		writeMinimalIntegrationManifest(t, d)
+		// No agent/stream directory — templates come entirely from the input package.
+		err := os.MkdirAll(filepath.Join(d, "data_stream", "logs"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "manifest.yml"), []byte(`
+streams:
+  - package: some_input_package
+    title: Composable
+    description: d
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
+		require.Empty(t, errs)
+	})
+
+	t.Run("composable stream with explicit template_paths validates those files", func(t *testing.T) {
+		d := t.TempDir()
+		writeMinimalIntegrationManifest(t, d)
+		err := os.MkdirAll(filepath.Join(d, "data_stream", "logs", "agent", "stream"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", "overlay.yml.hbs"), []byte(`x`), 0o644)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "manifest.yml"), []byte(`
+streams:
+  - package: some_input_package
+    title: Composable with overlay
+    description: d
+    template_paths:
+      - overlay.yml.hbs
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
+		require.Empty(t, errs)
+	})
+
+	t.Run("composable stream with explicit template_paths fails when file missing", func(t *testing.T) {
+		d := t.TempDir()
+		writeMinimalIntegrationManifest(t, d)
+		err := os.MkdirAll(filepath.Join(d, "data_stream", "logs", "agent", "stream"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "manifest.yml"), []byte(`
+streams:
+  - package: some_input_package
+    title: Composable with overlay
+    description: d
+    template_paths:
+      - missing.yml.hbs
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
+		require.Len(t, errs, 1)
+		require.Contains(t, errs[0].Error(), "template file not found")
+	})
 }
 func TestValidateIntegrationPolicyTemplates_NonIntegrationType(t *testing.T) {
 	d := t.TempDir()
@@ -323,6 +382,70 @@ streams:
 	errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
 	require.Empty(t, errs)
 }
+func TestValidateIntegrationPolicyTemplates_ComposableInputs(t *testing.T) {
+	t.Run("composable input with no templates skips validation", func(t *testing.T) {
+		d := t.TempDir()
+		err := os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: integration
+policy_templates:
+  - name: pt
+    inputs:
+      - package: some_input_package
+        title: Composable
+        description: d
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
+		require.Empty(t, errs)
+	})
+
+	t.Run("composable input with explicit template_paths validates those files", func(t *testing.T) {
+		d := t.TempDir()
+		err := os.MkdirAll(filepath.Join(d, "agent", "input"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "agent", "input", "overlay.yml.hbs"), []byte(`x`), 0o644)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: integration
+policy_templates:
+  - name: pt
+    inputs:
+      - package: some_input_package
+        title: Composable with overlay
+        description: d
+        template_paths:
+          - overlay.yml.hbs
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
+		require.Empty(t, errs)
+	})
+
+	t.Run("composable input with explicit template_paths fails when file missing", func(t *testing.T) {
+		d := t.TempDir()
+		err := os.MkdirAll(filepath.Join(d, "agent", "input"), 0o755)
+		require.NoError(t, err)
+		err = os.WriteFile(filepath.Join(d, "manifest.yml"), []byte(`
+type: integration
+policy_templates:
+  - name: pt
+    inputs:
+      - package: some_input_package
+        title: Composable with overlay
+        description: d
+        template_paths:
+          - missing.yml.hbs
+`), 0o644)
+		require.NoError(t, err)
+
+		errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
+		require.Len(t, errs, 1)
+		require.Contains(t, errs[0].Error(), "template file not found")
+	})
+}
+
 func TestFindPathAtDirectory(t *testing.T) {
 	d := t.TempDir()
 

code/go/internal/validator/semantic/validate_no_dev_folder.go

@@ -0,0 +1,39 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"io/fs"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
+)
+
+// ValidateNoDevFolder errors for any _dev/ directory found in the package.
+// _dev/ is a source-only artifact used during development (tests, deploy
+// configs, build manifests). It must not appear in a built package that is
+// validated with ModeBuild.
+func ValidateNoDevFolder(fsys fspath.FS) specerrors.ValidationErrors {
+	var errs specerrors.ValidationErrors
+	walkErr := fs.WalkDir(fsys, ".", func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() && d.Name() == "_dev" {
+			errs = append(errs, specerrors.NewStructuredErrorf(
+				"file %q: _dev directory is not allowed in built packages",
+				fsys.Path(p),
+			))
+			// Skip the subtree to avoid generating child errors for each
+			// file inside the _dev directory.
+			return fs.SkipDir
+		}
+		return nil
+	})
+	if walkErr != nil {
+		errs = append(errs, specerrors.NewStructuredError(walkErr, specerrors.UnassignedCode))
+	}
+	return errs
+}