feat: Resolve available, non-secret environment variables during compile

documentation/docs/configuration-default.md

@@ -108,10 +108,7 @@ The connector templates configured under `connectors` can use environment variab
 
 ### Environment Variables
 
-You can reference environment variables using the `${VAR_NAME}` placeholder syntax, for example `${POSTGRES_PASSWORD}`.
-At runtime, these placeholders are automatically resolved using the environment variables defined in the system or deployment environment.
-
-This can help decouple security credentials or add flexibility across different deployment environments.
+See [Environment Variables](configuration#environment-variables-var) for supported placeholder syntax and resolution behavior.
 
 ### SQRL Variables
 

documentation/docs/configuration.md

@@ -164,7 +164,16 @@ Refer to the individual engine configuration for connector configuration options
 
 ## Environment Variables (`${VAR}`)
 
-Environment variables (e.g. `${POSTGRES_PASSWORD}`) can be referenced inside the configuration files and SQRL scripts. Those are dynamically resolved by the DataSQRL runner when the pipeline is launched. If an environment variable is not configured, it is not replaced.
+Environment variables can be referenced with two placeholder types:
 
+- `${VAR_NAME}` references a non-secret environment variable, for example `${POSTGRES_HOST}`. DataSQRL treats this syntax as non-secret, even if the variable name contains words like `PASSWORD` or `TOKEN`.
+- `${{VAR_NAME}}` references a secret environment variable, for example `${{POSTGRES_PASSWORD}}`. Secret placeholders are not resolved during compile and are converted to `${VAR_NAME}` in generated artifacts so the actual value is resolved only when the pipeline runs.
 
+DataSQRL resolves environment variables at two different times:
 
+- During `sqrl compile`, DataSQRL resolves available, non-secret environment variables in user-provided `package.json` [connector configuration](configuration-default) and in Flink `CREATE TABLE ... WITH (...)` options in SQRL scripts.
+- If a `${VAR_NAME}` placeholder is not available during `sqrl compile`, it is left unchanged in the generated build artifacts so it can still be supplied later.
+- During `sqrl run` or `sqrl test`, DataSQRL first compiles the project with the same compile-time rules, then launches the generated artifacts and resolves remaining `${VAR_NAME}` placeholders defined anywhere in the SQRL scripts from the runtime environment.
+- If a required runtime placeholder is still unresolved when the generated artifact is launched, the run fails with an error identifying the missing variable.
+
+Compile-time values are written into the generated artifacts under `build/` and `build/deploy/`. Use the secret placeholder syntax for values that must not be resolved or written during compile.

pom.xml

@@ -167,6 +167,7 @@
     <snowflake-jdbc.version>4.2.0</snowflake-jdbc.version>
     <snakeyaml.version>2.6</snakeyaml.version>
     <swagger-core.version>2.2.50</swagger-core.version>
+    <system-stubs.version>2.1.8</system-stubs.version>
     <testcontainers.version>2.0.5</testcontainers.version>
     <vertx.version>5.0.12</vertx.version>
 
@@ -656,6 +657,12 @@
         <version>${testcontainers.version}</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>uk.org.webcompere</groupId>
+        <artifactId>system-stubs-jupiter</artifactId>
+        <version>${system-stubs.version}</version>
+        <scope>test</scope>
+      </dependency>
 
       <!-- logging -->
       <dependency>

sqrl-planner/pom.xml

@@ -133,6 +133,10 @@
       <artifactId>flink-avro</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.datasqrl.flinkrunner</groupId>
+      <artifactId>env-utils</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.datasqrl.flinkrunner</groupId>
       <artifactId>stdlib-utils</artifactId>

sqrl-planner/src/main/java/com/datasqrl/engine/stream/flink/plan/FlinkSqlNodeFactory.java

@@ -16,10 +16,11 @@
 package com.datasqrl.engine.stream.flink.plan;
 
 import com.datasqrl.calcite.schema.sql.SqlDataTypeSpecBuilder;
+import com.datasqrl.planner.util.NonSecretEnvVarResolver;
 import com.datasqrl.sql.SqlCallRewriter;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -42,6 +43,7 @@
 import org.apache.calcite.sql.parser.SqlParserPos;
 import org.apache.flink.sql.parser.ddl.SqlCreateFunction;
 import org.apache.flink.sql.parser.ddl.SqlCreateTable;
+import org.apache.flink.sql.parser.ddl.SqlCreateTableLike;
 import org.apache.flink.sql.parser.ddl.SqlCreateView;
 import org.apache.flink.sql.parser.ddl.SqlDistribution;
 import org.apache.flink.sql.parser.ddl.SqlTableColumn;
@@ -152,6 +154,40 @@ public static SqlTableConstraint createPrimaryKeyConstraint(List<String> primary
         SqlParserPos.ZERO);
   }
 
+  public static SqlCreateTable resolveTableProperties(SqlCreateTable createTable) {
+    var resolvedPropMap = resolvePropertiesToMap(createTable.getPropertyList());
+    var resolvedPropList = createProperties(resolvedPropMap);
+
+    if (createTable instanceof SqlCreateTableLike likeTable) {
+      return new SqlCreateTableLike(
+          likeTable.getParserPosition(),
+          likeTable.getTableName(),
+          likeTable.getColumnList(),
+          likeTable.getTableConstraints(),
+          resolvedPropList,
+          likeTable.getDistribution(),
+          likeTable.getPartitionKeyList(),
+          likeTable.getWatermark().orElse(null),
+          likeTable.getComment().orElse(null),
+          likeTable.getTableLike(),
+          likeTable.isTemporary(),
+          likeTable.ifNotExists);
+    }
+
+    return new SqlCreateTable(
+        createTable.getParserPosition(),
+        createTable.getTableName(),
+        createTable.getColumnList(),
+        createTable.getTableConstraints(),
+        resolvedPropList,
+        createTable.getDistribution(),
+        createTable.getPartitionKeyList(),
+        createTable.getWatermark().orElse(null),
+        createTable.getComment().orElse(null),
+        createTable.isTemporary(),
+        createTable.ifNotExists);
+  }
+
   public static SqlNodeList createProperties(Map<String, String> options) {
     List<SqlNode> props =
         options.entrySet().stream()
@@ -167,15 +203,20 @@ public static SqlNodeList createProperties(Map<String, String> options) {
     return new SqlNodeList(props, SqlParserPos.ZERO);
   }
 
-  public static Map<String, String> propertiesToMap(SqlNodeList nodeList) {
-    Map<String, String> result = new HashMap<>();
-    for (SqlNode node : nodeList) {
+  public static Map<String, String> resolvePropertiesToMap(SqlNodeList nodeList) {
+    var res = new LinkedHashMap<String, String>();
+    var resolver = NonSecretEnvVarResolver.builder().strict(false).build();
+
+    for (var node : nodeList) {
       var option = (SqlTableOption) node;
       var keyLiteral = (SqlLiteral) option.getKey();
       var valueLiteral = (SqlLiteral) option.getValue();
-      result.put(keyLiteral.toValue(), valueLiteral.toValue());
+      var resolvedVal = resolver.resolve(valueLiteral.toValue());
+
+      res.put(keyLiteral.toValue(), resolvedVal);
     }
-    return result;
+
+    return res;
   }
 
   public static SqlNodeList createPartitionKeys(List<String> partitionKeys) {

sqrl-planner/src/main/java/com/datasqrl/planner/Sqrl2FlinkSQLTranslator.java

@@ -828,7 +828,7 @@ private AddTableResult addTable(
       Optional<MutationBuilder> mutationBuilder) {
     var tableSqlNode = parseSQL(createTableSql);
     checkArgument(tableSqlNode instanceof SqlCreateTable, "Expected CREATE TABLE statement");
-    var tableDefinition = (SqlCreateTable) tableSqlNode;
+    var tableDefinition = FlinkSqlNodeFactory.resolveTableProperties((SqlCreateTable) tableSqlNode);
     var fullTable = tableDefinition;
     var origTableName = fullTable.getTableName().getSimple();
     final var finalTableName = tableNameModifier.apply(origTableName);
@@ -837,7 +837,7 @@ private AddTableResult addTable(
       // Check if the LIKE clause is referencing an external schema
       SqlTableLike likeClause = likeTable.getTableLike();
       var likeTableName = likeClause.getSourceTable().toString();
-      var likeTableProps = FlinkSqlNodeFactory.propertiesToMap(likeTable.getPropertyList());
+      var likeTableProps = FlinkSqlNodeFactory.resolvePropertiesToMap(likeTable.getPropertyList());
       Optional<SchemaConversionResult> schema =
           schemaLoader.loadSchema(finalTableName, likeTableName, likeTableProps);
       if (schema.isPresent()) {

sqrl-planner/src/main/java/com/datasqrl/planner/tables/FlinkTableBuilder.java

@@ -130,7 +130,7 @@ public FlinkTableBuilder setConnectorOptions(Map<String, String> options) {
   }
 
   public Map<String, String> getConnectorOptions() {
-    return FlinkSqlNodeFactory.propertiesToMap(getPropertyList());
+    return FlinkSqlNodeFactory.resolvePropertiesToMap(getPropertyList());
   }
 
   public FlinkTableBuilder setDummyConnector() {

sqrl-planner/src/main/java/com/datasqrl/planner/util/NonSecretEnvVarResolver.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright © 2021 DataSQRL (contact@datasqrl.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.datasqrl.planner.util;
+
+import com.datasqrl.flinkrunner.utils.EnvVarResolver;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import lombok.experimental.SuperBuilder;
+
+/** Resolves regular environment variables and converts secret templates to standard templates. */
+@SuperBuilder
+public class NonSecretEnvVarResolver extends EnvVarResolver {
+
+  private static final Pattern SECRET_ENVIRONMENT_VARIABLE_PATTERN =
+      Pattern.compile("\\$\\{\\{(.*?)\\}\\}");
+
+  private static final Pattern SECRET_ENVIRONMENT_VARIABLE_NAME_PATTERN =
+      Pattern.compile("[A-Za-z_][A-Za-z0-9_]*");
+
+  @Override
+  public String resolve(String src) {
+    if (src == null || src.isBlank()) {
+      return src;
+    }
+
+    var resolvedRegularEnvVars = super.resolve(src);
+
+    return resolveSecretEnvVars(resolvedRegularEnvVars);
+  }
+
+  private String resolveSecretEnvVars(String src) {
+    var res = new StringBuilder();
+    var matcher = SECRET_ENVIRONMENT_VARIABLE_PATTERN.matcher(src);
+    while (matcher.find()) {
+      var key = matcher.group(1);
+      if (!SECRET_ENVIRONMENT_VARIABLE_NAME_PATTERN.matcher(key).matches()) {
+        throw new IllegalArgumentException(
+            "Secret environment variable templates only support variable names, but found: %s"
+                .formatted(matcher.group()));
+      }
+
+      matcher.appendReplacement(res, Matcher.quoteReplacement(String.format("${%s}", key)));
+    }
+    matcher.appendTail(res);
+
+    return res.toString();
+  }
+}

sqrl-planner/src/main/java/com/datasqrl/util/ConfigLoaderUtils.java

@@ -26,8 +26,10 @@
 import com.datasqrl.error.ErrorMessage;
 import com.datasqrl.error.ResourceFileUtil;
 import com.datasqrl.planner.dag.plan.MutationDatabase;
+import com.datasqrl.planner.util.NonSecretEnvVarResolver;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.annotations.VisibleForTesting;
 import com.networknt.schema.SchemaRegistry;
@@ -101,7 +103,9 @@ static PackageJson loadResolvedConfigFromFile(
         Files.isRegularFile(packageJson),
         String.format("Failed to load %s, it is not a regular file", packageJson));
 
-    var objectNode = convertFileToObjectNode(errors, Either.Left(packageJson));
+    // This is called by 'run', 'test', and 'exec', so env var resolution will be done by
+    // DatasqrlRun.
+    var objectNode = convertFileToObjectNode(errors, Either.Left(packageJson), false);
 
     return SqrlConfig.loadResolvedConfig(errors, objectNode, packageSchemaPath);
   }
@@ -294,14 +298,14 @@ static PackageJson loadUnresolvedConfig(
       if (url == null) {
         throw errors.withConfig(defaultPath).exception("Default configuration not found");
       }
-      var defaultJson = convertFileToObjectNode(errors, Either.Right(url));
+      var defaultJson = convertFileToObjectNode(errors, Either.Right(url), false);
       valid &= isValidJson(errors, defaultJson, PACKAGE_SCHEMA_PATH);
       jsons.add(defaultJson);
     }
 
     // Convert, validate, and add files
     for (Path file : files) {
-      var json = convertFileToObjectNode(errors, Either.Left(file));
+      var json = convertFileToObjectNode(errors, Either.Left(file), true);
       valid &= isValidJson(errors, json, PACKAGE_SCHEMA_PATH);
       jsons.add(json);
     }
@@ -326,15 +330,60 @@ private static void validatePlanDir(Path planDir) {
         Files.isDirectory(planDir), "Failed to load Flink config, plan dir does not exist.");
   }
 
-  private static ObjectNode convertFileToObjectNode(ErrorCollector errors, Either<Path, URL> file) {
-    var local = errors.withConfig(file.isLeft() ? file.left().toString() : file.right().toString());
+  private static ObjectNode convertFileToObjectNode(
+      ErrorCollector errors, Either<Path, URL> file, boolean resolveEnvVars) {
+    var filename = file.isLeft() ? file.left().toString() : file.right().toString();
+    var localErr = errors.withConfig(filename);
+
     try {
-      var jsonNode =
-          file.isLeft() ? MAPPER.readTree(file.left().toFile()) : MAPPER.readTree(file.right());
+      JsonNode jsonNode;
+      if (file.isLeft()) {
+        jsonNode = MAPPER.readTree(file.left().toFile());
+      } else {
+        jsonNode = MAPPER.readTree(file.right().openStream());
+      }
+
+      if (resolveEnvVars) {
+        var resolver = NonSecretEnvVarResolver.builder().strict(false).build();
+        resolveConnectorEnvVars(jsonNode.get("connectors"), resolver);
+      }
+
       return (ObjectNode) jsonNode;
 
     } catch (IOException e) {
-      throw local.exception("Could not parse JSON file [%s]: %s", file, e.toString());
+      throw localErr.exception("Could not parse JSON file [%s]: %s", file, e.toString());
+    }
+  }
+
+  private static void resolveConnectorEnvVars(JsonNode node, NonSecretEnvVarResolver resolver) {
+    if (node == null) {
+      return;
+    }
+
+    if (node.isObject()) {
+      var objectNode = (ObjectNode) node;
+      for (var field : objectNode.properties()) {
+        var value = field.getValue();
+
+        if (value.isContainerNode()) {
+          resolveConnectorEnvVars(value, resolver);
+
+        } else if (value.isTextual()) {
+          objectNode.put(field.getKey(), resolver.resolve(value.asText()));
+        }
+      }
+    } else if (node.isArray()) {
+      var arrayNode = (ArrayNode) node;
+      for (var i = 0; i < arrayNode.size(); i++) {
+        var value = arrayNode.get(i);
+
+        if (value.isContainerNode()) {
+          resolveConnectorEnvVars(value, resolver);
+
+        } else if (value.isTextual()) {
+          arrayNode.set(i, MAPPER.getNodeFactory().textNode(resolver.resolve(value.asText())));
+        }
+      }
     }
   }
 }