Support SQLView resources and view composition in SQL on FHIR operations

deployment/helm/pathling/templates/role.yaml

@@ -6,6 +6,11 @@ rules:
   - apiGroups: [""]
     resources: ["pods", "services", "configmaps"]
     verbs: ["create", "get", "list", "watch", "delete"]
+  # Allow the driver to provision and remove the dynamically created scratch
+  # volumes used for executor local storage (claimName OnDemand).
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["create", "get", "list", "watch", "delete"]
   - apiGroups: [""]
     resources: ["pods/log"]
     verbs: ["get", "list", "watch"]

encoders/src/main/java/au/csiro/pathling/encoders/ViewDefinitionResource.java

@@ -73,6 +73,14 @@ public class ViewDefinitionResource extends DomainResource {
 
   @Serial private static final long serialVersionUID = 1909997123685548098L;
 
+  @Nullable
+  @Child(name = "url")
+  private UriType url;
+
+  @Nullable
+  @Child(name = "version")
+  private StringType version;
+
   @Nullable
   @Getter
   @Child(name = "name")
@@ -100,6 +108,50 @@ public class ViewDefinitionResource extends DomainResource {
   @Child(name = "constant", max = Child.MAX_UNLIMITED)
   private List<ConstantComponent> constant;
 
+  @Nullable
+  public String getUrl() {
+    return url == null ? null : url.getValue();
+  }
+
+  @Nullable
+  public UriType getUrlElement() {
+    return url;
+  }
+
+  public boolean hasUrlElement() {
+    return url != null && !url.isEmpty();
+  }
+
+  public void setUrlElement(final UriType url) {
+    this.url = url;
+  }
+
+  public void setUrl(final String url) {
+    this.url = url == null ? null : new UriType(url);
+  }
+
+  @Nullable
+  public String getVersion() {
+    return version == null ? null : version.getValue();
+  }
+
+  @Nullable
+  public StringType getVersionElement() {
+    return version;
+  }
+
+  public boolean hasVersionElement() {
+    return version != null && !version.isEmpty();
+  }
+
+  public void setVersionElement(final StringType version) {
+    this.version = version;
+  }
+
+  public void setVersion(final String version) {
+    this.version = version == null ? null : new StringType(version);
+  }
+
   @Nullable
   public StringType getNameElement() {
     return name;
@@ -187,6 +239,8 @@ public boolean hasConstant() {
   public DomainResource copy() {
     final ViewDefinitionResource copy = new ViewDefinitionResource();
     copyValues(copy);
+    copy.url = url != null ? url.copy() : null;
+    copy.version = version != null ? version.copy() : null;
     copy.name = name != null ? name.copy() : null;
     if (fhirVersion != null) {
       copy.fhirVersion = new ArrayList<>();
@@ -233,6 +287,8 @@ public String fhirType() {
   @Override
   public boolean isEmpty() {
     return super.isEmpty()
+        && (url == null || url.isEmpty())
+        && (version == null || version.isEmpty())
         && (name == null || name.isEmpty())
         && (fhirVersion == null || fhirVersion.isEmpty())
         && (resource == null || resource.isEmpty())

encoders/src/test/java/au/csiro/pathling/encoders/ViewDefinitionEncodingTest.java

@@ -55,6 +55,7 @@
 import org.hl7.fhir.r4.model.BooleanType;
 import org.hl7.fhir.r4.model.CodeType;
 import org.hl7.fhir.r4.model.IntegerType;
+import org.hl7.fhir.r4.model.UriType;
 import org.json.JSONException;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
@@ -127,6 +128,8 @@ void testSchemaHasBasicFields() {
 
     // Verify top-level fields exist.
     assertTrue(schema.getFieldIndex("id").isDefined());
+    assertTrue(schema.getFieldIndex("url").isDefined());
+    assertTrue(schema.getFieldIndex("version").isDefined());
     assertTrue(schema.getFieldIndex("name").isDefined());
     assertTrue(schema.getFieldIndex("resource").isDefined());
     assertTrue(schema.getFieldIndex("status").isDefined());
@@ -135,6 +138,49 @@ void testSchemaHasBasicFields() {
     assertTrue(schema.getFieldIndex("constant").isDefined());
   }
 
+  @Test
+  void testUrlAndVersionSurviveRoundTrip() {
+    // The url and version are required to match dependency references by canonical URL, so they
+    // must be retained through an encode/decode round-trip.
+    final ViewDefinitionResource original = createSimpleViewDefinition();
+    original.setUrlElement(new UriType("https://example.org/ViewDefinition/patients"));
+    original.setVersionElement(new org.hl7.fhir.r4.model.StringType("2.0"));
+
+    final ExpressionEncoder<ViewDefinitionResource> encoder =
+        fhirEncodersL0.of(ViewDefinitionResource.class);
+    final ExpressionEncoder<ViewDefinitionResource> resolvedEncoder =
+        EncoderUtils.defaultResolveAndBind(encoder);
+
+    final InternalRow serializedRow = resolvedEncoder.createSerializer().apply(original);
+    final ViewDefinitionResource decoded =
+        resolvedEncoder.createDeserializer().apply(serializedRow);
+
+    assertTrue(original.equalsDeep(decoded));
+    assertEquals("https://example.org/ViewDefinition/patients", decoded.getUrl());
+    assertEquals("2.0", decoded.getVersion());
+  }
+
+  @Test
+  void testDecodeWithoutUrlAndVersionYieldsEmptyValues() {
+    // A ViewDefinition stored without a url or version (the pre-existing case) must decode cleanly
+    // with absent url and version, leaving it unmatchable by canonical URL.
+    final ViewDefinitionResource original = createSimpleViewDefinition();
+
+    final ExpressionEncoder<ViewDefinitionResource> encoder =
+        fhirEncodersL0.of(ViewDefinitionResource.class);
+    final ExpressionEncoder<ViewDefinitionResource> resolvedEncoder =
+        EncoderUtils.defaultResolveAndBind(encoder);
+
+    final InternalRow serializedRow = resolvedEncoder.createSerializer().apply(original);
+    final ViewDefinitionResource decoded =
+        resolvedEncoder.createDeserializer().apply(serializedRow);
+
+    assertFalse(decoded.hasUrlElement());
+    assertFalse(decoded.hasVersionElement());
+    assertNull(decoded.getUrl());
+    assertNull(decoded.getVersion());
+  }
+
   @Test
   void testSchemaHandlesRecursiveSelectComponent() {
     // Level 0: should NOT have nested select.select or unionAll.
@@ -846,4 +892,84 @@ void testFhirType() {
     final ViewDefinitionResource view = new ViewDefinitionResource();
     assertEquals("ViewDefinition", view.fhirType());
   }
+
+  // ========== URL AND VERSION ACCESSOR TESTS ==========
+
+  @Test
+  void testSetUrlWithStringValue() {
+    // setUrl(String) wraps the value in a UriType and getUrl() unwraps it.
+    final ViewDefinitionResource view = new ViewDefinitionResource();
+    view.setUrl("https://example.org/ViewDefinition/patients");
+
+    assertTrue(view.hasUrlElement());
+    assertInstanceOf(UriType.class, view.getUrlElement());
+    assertEquals("https://example.org/ViewDefinition/patients", view.getUrl());
+  }
+
+  @Test
+  void testSetUrlWithNullValue() {
+    // setUrl(null) clears the element, leaving the field null so getUrl() returns null.
+    final ViewDefinitionResource view = new ViewDefinitionResource();
+    view.setUrl("https://example.org/ViewDefinition/patients");
+    view.setUrl(null);
+
+    assertNull(view.getUrlElement());
+    assertNull(view.getUrl());
+    assertFalse(view.hasUrlElement());
+  }
+
+  @Test
+  void testSetVersionWithStringValue() {
+    // setVersion(String) wraps the value in a StringType and getVersion() unwraps it.
+    final ViewDefinitionResource view = new ViewDefinitionResource();
+    view.setVersion("2.0");
+
+    assertTrue(view.hasVersionElement());
+    assertInstanceOf(org.hl7.fhir.r4.model.StringType.class, view.getVersionElement());
+    assertEquals("2.0", view.getVersion());
+  }
+
+  @Test
+  void testSetVersionWithNullValue() {
+    // setVersion(null) clears the element, leaving the field null so getVersion() returns null.
+    final ViewDefinitionResource view = new ViewDefinitionResource();
+    view.setVersion("2.0");
+    view.setVersion(null);
+
+    assertNull(view.getVersionElement());
+    assertNull(view.getVersion());
+    assertFalse(view.hasVersionElement());
+  }
+
+  @Test
+  void testCopyRetainsUrlAndVersion() {
+    // copy() must duplicate the url and version elements when they are present.
+    final ViewDefinitionResource original = createSimpleViewDefinition();
+    original.setUrl("https://example.org/ViewDefinition/patients");
+    original.setVersion("2.0");
+
+    final ViewDefinitionResource copy = (ViewDefinitionResource) original.copy();
+
+    assertTrue(original.equalsDeep(copy));
+    assertEquals("https://example.org/ViewDefinition/patients", copy.getUrl());
+    assertEquals("2.0", copy.getVersion());
+  }
+
+  @Test
+  void testIsEmptyConsidersUrlAndVersion() {
+    // A populated url or version makes the resource non-empty.
+    final ViewDefinitionResource withUrl = new ViewDefinitionResource();
+    withUrl.setUrl("https://example.org/ViewDefinition/patients");
+    assertFalse(withUrl.isEmpty());
+
+    final ViewDefinitionResource withVersion = new ViewDefinitionResource();
+    withVersion.setVersion("1.0");
+    assertFalse(withVersion.isEmpty());
+
+    // Non-null but empty url and version elements leave the resource empty.
+    final ViewDefinitionResource emptyElements = new ViewDefinitionResource();
+    emptyElements.setUrlElement(new UriType());
+    emptyElements.setVersionElement(new org.hl7.fhir.r4.model.StringType());
+    assertTrue(emptyElements.isEmpty());
+  }
 }

server/pom.xml

@@ -36,7 +36,7 @@
     <maven.compiler.target>21</maven.compiler.target>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.majorVersion>1</project.majorVersion>
-    <pathling.version>9.7.1</pathling.version>
+    <pathling.version>9.8.0</pathling.version>
     <pathling.hapiFhirVersion>8.10.0</pathling.hapiFhirVersion>
     <pathling.springBootVersion>3.5.14</pathling.springBootVersion>
     <pathling.sparkVersion>4.0.2</pathling.sparkVersion>
@@ -56,6 +56,17 @@
     <pathling.awsSdkVersion>2.40.3</pathling.awsSdkVersion>
     <pathling.bulkExportVersion>1.0.4</pathling.bulkExportVersion>
     <pathling.fhirAuthVersion>1.0.0</pathling.fhirAuthVersion>
+
+    <!-- The JVM module options required to run Pathling on Spark with Java 21.
+         These open the JDK internals that Spark accesses reflectively, most
+         notably sun.util.calendar, which Spark's date conversion reaches via a
+         method handle. They are referenced by the spring-boot:run goal and must
+         be kept in sync with the production entrypoint, which cannot reference
+         this property at runtime (see src/main/jib/usr/bin/entrypoint.sh). -->
+    <pathling.runtime.jvmModuleOpts>--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED</pathling.runtime.jvmModuleOpts>
+    <!-- The test harness needs every runtime option plus additional opens for
+         the mocking and serialisation frameworks exercised during testing. -->
+    <pathling.test.jvmModuleOpts>${pathling.runtime.jvmModuleOpts} --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED</pathling.test.jvmModuleOpts>
   </properties>
 
   <repositories>
@@ -585,7 +596,9 @@
         <artifactId>spring-boot-maven-plugin</artifactId>
         <version>${pathling.springBootVersion}</version>
         <configuration>
-          <jvmArguments>--add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED</jvmArguments>
+          <!-- Run with the shared runtime module options so that a local
+               spring-boot:run faithfully reproduces the production runtime. -->
+          <jvmArguments>${pathling.runtime.jvmModuleOpts}</jvmArguments>
         </configuration>
       </plugin>
       <plugin>
@@ -601,22 +614,7 @@
           <!-- Each integration test class runs in its own JVM to prevent URLStreamHandlerFactory
                conflicts between Spark/Hadoop and Tomcat. See SPARK-25694 and Spring Boot #10617. -->
           <reuseForks>false</reuseForks>
-          <argLine>@{argLine}
-            --add-opens java.base/java.lang=ALL-UNNAMED
-            --add-opens java.base/java.lang.invoke=ALL-UNNAMED
-            --add-opens java.base/java.lang.reflect=ALL-UNNAMED
-            --add-opens java.base/java.io=ALL-UNNAMED
-            --add-opens java.base/java.net=ALL-UNNAMED
-            --add-opens java.base/java.nio=ALL-UNNAMED
-            --add-opens java.base/java.util=ALL-UNNAMED
-            --add-opens java.base/java.util.concurrent=ALL-UNNAMED
-            --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED
-            --add-opens java.base/sun.nio.ch=ALL-UNNAMED
-            --add-opens java.base/sun.nio.cs=ALL-UNNAMED
-            --add-opens java.base/sun.security.action=ALL-UNNAMED
-            --add-opens java.base/sun.util.calendar=ALL-UNNAMED
-            --add-exports java.base/sun.nio.ch=ALL-UNNAMED
-          </argLine>
+          <argLine>@{argLine} ${pathling.test.jvmModuleOpts}</argLine>
         </configuration>
         <executions>
           <execution>
@@ -639,22 +637,7 @@
               junit.jupiter.execution.parallel.enabled=true
             </configurationParameters>
           </properties>
-          <argLine>@{argLine}
-            --add-opens java.base/java.lang=ALL-UNNAMED
-            --add-opens java.base/java.lang.invoke=ALL-UNNAMED
-            --add-opens java.base/java.lang.reflect=ALL-UNNAMED
-            --add-opens java.base/java.io=ALL-UNNAMED
-            --add-opens java.base/java.net=ALL-UNNAMED
-            --add-opens java.base/java.nio=ALL-UNNAMED
-            --add-opens java.base/java.util=ALL-UNNAMED
-            --add-opens java.base/java.util.concurrent=ALL-UNNAMED
-            --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED
-            --add-opens java.base/sun.nio.ch=ALL-UNNAMED
-            --add-opens java.base/sun.nio.cs=ALL-UNNAMED
-            --add-opens java.base/sun.security.action=ALL-UNNAMED
-            --add-opens java.base/sun.util.calendar=ALL-UNNAMED
-            --add-exports java.base/sun.nio.ch=ALL-UNNAMED
-          </argLine>
+          <argLine>@{argLine} ${pathling.test.jvmModuleOpts}</argLine>
         </configuration>
       </plugin>
       <plugin>

server/src/main/java/au/csiro/pathling/config/SqlQueryConfiguration.java

@@ -49,4 +49,14 @@ public class SqlQueryConfiguration {
    */
   @Min(1)
   private long timeoutSeconds = 60L;
+
+  /**
+   * The maximum nesting depth of the dependency graph resolved for a single query. The top-level
+   * query's direct {@code relatedArtifact} dependencies sit at depth one; each further level of
+   * nested {@code SQLView} dependency increments the depth. A graph that nests deeper than this
+   * limit is rejected before any Spark work, guarding against accidental fan-out and runaway
+   * resolution. Real view graphs are shallow, so the default is generous while still bounded.
+   */
+  @Min(1)
+  private int maxDependencyDepth = 10;
 }