LIVY-979: Create a helm-based integration environment for debugging Livy on Kubernetes

.rat-excludes

@@ -30,3 +30,4 @@ logs/*
 docs/**/*.html
 docs/**/JB/**
 venv/*
+dev/helmchart/Chart.lock

dev/docker/README.md

@@ -1,4 +1,7 @@
 # Livy with standalone Spark Cluster
+
+For a local Kubernetes development environment, see [../helmchart/README.md](../helmchart/README.md).
+
 ## Pre-requisite
 Following steps use Ubuntu as development environment but most of the instructions can be modified to fit another OS as well.
 * Install wsl if on windows, instructions available [here](https://ubuntu.com/tutorials/install-ubuntu-on-wsl2-on-windows-11-with-gui-support)

dev/helmchart/.gitignore

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+charts/*.tgz

dev/helmchart/.helmignore

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+.DS_Store
+.git/
+.idea/
+*.swp
+*.bak
+*.tmp
+test/

dev/helmchart/Chart.lock

@@ -0,0 +1,9 @@
+dependencies:
+- name: ingress-nginx
+  repository: https://kubernetes.github.io/ingress-nginx
+  version: 4.12.1
+- name: cert-manager
+  repository: https://charts.jetstack.io
+  version: v1.16.2
+digest: sha256:787a1aa996a76ef0b6f327fdabc51f473ff0708890c3d7f57b95d165607568a5
+generated: "2026-08-06T11:09:10.754267+05:30"

dev/helmchart/Chart.yaml

@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v2
+name: livycluster
+description: Local Apache Livy and Spark development environment on Kubernetes
+type: application
+version: 0.1.0
+appVersion: "1.0.0-SNAPSHOT"
+home: https://livy.apache.org/
+sources:
+  - https://github.com/apache/livy
+maintainers:
+  - name: Apache Livy
+    email: livy@apache.org
+keywords:
+  - livy
+  - spark
+  - kubernetes
+dependencies:
+  - name: ingress-nginx
+    version: 4.12.1
+    repository: https://kubernetes.github.io/ingress-nginx
+    condition: ingress-nginx.enabled
+    tags:
+      - ingress
+  - name: cert-manager
+    version: v1.16.2
+    repository: https://charts.jetstack.io
+    condition: cert-manager.enabled
+    tags:
+      - ingress

dev/helmchart/Docker.md

@@ -0,0 +1,80 @@
+# Building Container Images for the Kubernetes Helm Chart
+
+These steps build Spark and Livy images for the local Kubernetes development environment
+described in [README.md](README.md). Version defaults match `values.yaml`.
+
+## Build Livy
+
+From the repository root:
+
+```shell
+mvn -Pthriftserver -Pscala-2.12 -Pspark3 package
+cp assembly/target/apache-livy-1.0.0-SNAPSHOT_2.12-bin.zip /tmp/
+```
+
+## Spark image with Python bindings
+
+Download and extract Spark:
+
+```shell
+wget https://archive.apache.org/dist/spark/spark-3.5.6/spark-3.5.6-bin-hadoop3.tgz
+tar -xzf spark-3.5.6-bin-hadoop3.tgz
+```
+
+Build and push the Spark image:
+
+```shell
+./spark-3.5.6-bin-hadoop3/bin/docker-image-tool.sh \
+  -r <your_repository> \
+  -t v3.5.6 \
+  -p kubernetes/dockerfiles/spark/bindings/python/Dockerfile \
+  build
+
+./spark-3.5.6-bin-hadoop3/bin/docker-image-tool.sh \
+  -r <your_repository> \
+  -t v3.5.6 \
+  -p kubernetes/dockerfiles/spark/bindings/python/Dockerfile \
+  push
+```
+
+## Livy image
+
+Create `/tmp/Dockerfile`:
+
+```dockerfile
+FROM <your_repository>/spark-py:v3.5.6
+
+ENV LIVY_VERSION=1.0.0-SNAPSHOT
+ENV LIVY_PACKAGE=apache-livy-${LIVY_VERSION}_2.12-bin
+ENV LIVY_HOME=/opt/livy
+ENV LIVY_CONF_DIR=/conf
+ENV PATH=$PATH:$LIVY_HOME/bin
+
+USER root
+
+COPY ${LIVY_PACKAGE}.zip /
+RUN apt-get update && apt-get install -y unzip && \
+    unzip /${LIVY_PACKAGE}.zip -d / && \
+    mv /${LIVY_PACKAGE} /opt/ && \
+    rm -rf ${LIVY_HOME} && \
+    ln -s /opt/${LIVY_PACKAGE} ${LIVY_HOME} && \
+    rm -f /${LIVY_PACKAGE}.zip
+
+RUN mkdir -p /var/log/livy && ln -s /var/log/livy ${LIVY_HOME}/logs
+
+WORKDIR ${LIVY_HOME}
+ENTRYPOINT ["livy-server"]
+```
+
+Build and push:
+
+```shell
+cd /tmp
+docker build -t <your_repository>/livy:spark3.5.6 .
+docker push <your_repository>/livy:spark3.5.6
+rm -f apache-livy-1.0.0-SNAPSHOT_2.12-bin.zip
+```
+
+Update `values.yaml` or pass `--set` flags so `image.livy.repository`,
+`image.livy.tag`, `image.spark.repository`, and `image.spark.tag` match the images
+you pushed.

dev/helmchart/README.md

@@ -0,0 +1,161 @@
+# Apache Livy on Kubernetes (Local Development)
+
+This Helm chart deploys Apache Livy and Apache Spark on a local Kubernetes cluster,
+such as Docker Desktop with Kubernetes enabled. It is intended for development and
+debugging without relying on cloud services.
+
+JIRA: [LIVY-979](https://issues.apache.org/jira/browse/LIVY-979)
+
+## Prerequisites
+
+1. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/) and enable Kubernetes.
+2. Install [Helm 3](https://helm.sh/docs/intro/install/).
+3. Build and push the Spark and Livy container images described in [Docker.md](Docker.md).
+4. Add a hosts entry that matches `clusterHost` in `values.yaml` (default `my-cluster.example.com`):
+
+```text
+127.0.0.1 my-cluster.example.com
+```
+
+## Version alignment
+
+The chart defaults match a Livy build produced with:
+
+```shell
+mvn -Pthriftserver -Pscala-2.12 -Pspark3 package
+```
+
+| Setting | Default |
+|---------|---------|
+| Livy | `1.0.0-SNAPSHOT` |
+| Spark | `3.5.6` |
+| Scala | `2.12` |
+
+Update `values.yaml` if you build with different profiles or versions. Image tags default
+from `sparkVersion` when left empty (`v3.5.6` for Spark, `spark3.5.6` for Livy).
+
+## Build chart dependencies
+
+Chart dependencies are declared in `Chart.yaml` and resolved at deploy time. Do not
+commit the downloaded archives under `charts/`:
+
+```shell
+cd dev/helmchart
+helm dependency build
+```
+
+When ingress is enabled, the chart installs `cert-manager` and `ingress-nginx` as
+subcharts. On a fresh cluster, cert-manager CRDs are installed via
+`cert-manager.installCRDs=true` (the chart default).
+
+References:
+- [cert-manager nginx ingress tutorial](https://cert-manager.io/docs/tutorials/acme/nginx-ingress/)
+- [cert-manager Helm chart](https://artifacthub.io/packages/helm/cert-manager/cert-manager)
+
+## Deploy the chart
+
+```shell
+cd dev/helmchart
+helm dependency build
+
+kubectl create namespace livy-dev
+
+helm -n livy-dev install livycluster . \
+  --set image.livy.repository=<your_repository>/livy \
+  --set image.spark.repository=<your_repository>/spark-py \
+  --set ingress-nginx.controller.extraArgs.default-ssl-certificate=livy-dev/ingress-default-tls
+```
+
+Set `ingress-nginx.controller.extraArgs.default-ssl-certificate` to
+`<release-namespace>/<tls-secret-name>` (defaults: `livy-dev/ingress-default-tls`).
+
+## Remote debugging
+
+Remote debugging is enabled by default (`debug.enabled: true`) on port `9010`, matching
+the pattern used in `dev/docker/livy-dev-cluster/conf/livy/livy-env.sh`. Disable it for
+non-debug deployments:
+
+```shell
+helm upgrade livycluster . -n livy-dev --set debug.enabled=false
+```
+
+## Grafana and Loki (optional)
+
+Livy can link Spark driver and executor logs to Grafana when Loki is available. The chart
+maps the following `values.yaml` keys to `livy.conf`:
+
+| values.yaml | livy.conf |
+|-------------|-----------|
+| `grafana.loki.enabled` | `livy.server.kubernetes.grafana.loki.enabled` |
+| `grafana.url` | `livy.server.kubernetes.grafana.url` |
+| `grafana.lokiDatasource` | `livy.server.kubernetes.grafana.loki.datasource` |
+| `grafana.timeRange` | `livy.server.kubernetes.grafana.timeRange` |
+
+To enable Loki integration:
+
+1. Install Grafana and Loki in the cluster (for example with the [Loki stack Helm chart](https://grafana.com/docs/loki/latest/setup/install/helm/)).
+2. Add a Loki datasource named `loki` in Grafana (or override `grafana.lokiDatasource`).
+3. Deploy or upgrade with Loki integration enabled:
+
+```shell
+helm upgrade livycluster . -n livy-dev \
+  --set grafana.loki.enabled=true \
+  --set grafana.url=http://grafana.livy-dev.svc.cluster.local:3000
+```
+
+When enabled, Livy adds log links in the session UI that open Grafana Explore for the
+matching Spark application labels.
+
+## Verify the deployment
+
+```shell
+kubectl -n livy-dev get pods -w
+```
+
+### REST API smoke tests
+
+Create an interactive session:
+
+```shell
+curl -k -X POST -H "Content-Type: application/json" \
+  --data '{"kind": "spark"}' \
+  https://my-cluster.example.com/livy/sessions | jq
+```
+
+Run a statement:
+
+```shell
+curl -k -X POST \
+  -H "Content-Type: application/json" \
+  -d '{"kind": "spark", "code": "sc.parallelize(1 to 10).count()"}' \
+  https://my-cluster.example.com/livy/sessions/0/statements | jq
+```
+
+Submit a batch job:
+
+```shell
+curl -s -k -H "Content-Type: application/json" -X POST \
+  -d '{
+    "name": "testbatch1",
+    "className": "org.apache.spark.examples.SparkPi",
+    "numExecutors": 2,
+    "file": "local:///opt/spark/examples/jars/spark-examples_2.12-3.5.6.jar",
+    "args": ["10000"]
+  }' "https://my-cluster.example.com/livy/batches" | jq
+```
+
+Update the examples JAR path if you change `scalaBinaryVersion` or `sparkVersion` in
+`values.yaml` (pattern: `spark-examples_<scala>-<spark>.jar`).
+
+## Chart validation
+
+Run the chart checks locally:
+
+```shell
+dev/helmchart/test/validate.sh
+```
+
+## Related documentation
+
+- Docker image build steps: [Docker.md](Docker.md)
+- Standalone Spark cluster (non-Kubernetes): [../docker/README.md](../docker/README.md)

dev/helmchart/templates/NOTES.txt

@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+The Livy server has been installed.
+Check its status by running:
+  kubectl -n {{ .Release.Namespace }} get pods -w
+Connect to the Livy Web UI:
+{{- if .Values.ingress.enabled }}
+    # Open in browser:
+    #  https://{{ .Values.clusterHost }}/livy/ui
+    #
+    # If TLS fails on first install, set ingress-nginx default certificate to:
+    #  {{ include "livycluster.defaultTlsCertificate" . }}
+{{- else }}
+    kubectl -n {{ .Release.Namespace }} port-forward {{ include "livycluster.fullname" . }}-0 8998
+    # Open in browser: http://localhost:8998
+{{- end }}
+{{- if .Values.debug.enabled }}
+
+Remote debugging is enabled on port {{ .Values.debug.port }}.
+{{- end }}
+{{- if .Values.grafana.loki.enabled }}
+
+Grafana Loki integration is enabled. Open {{ .Values.grafana.url }} and use datasource
+"{{ .Values.grafana.lokiDatasource }}" to inspect Spark driver and executor logs.
+{{- end }}