NVIDIA · sjenning · Jun 24, 2026 · Jun 24, 2026
@@ -192,6 +192,15 @@ release. Look for failed installs, unexpected values, missing namespace, wrong
 image tag, TLS settings that do not match the registered endpoint, and
 scheduling failures.
 
+When no external credential driver is enabled, the Helm chart uses the
+gateway's default encrypted database credential storage. The chart creates a
+retained Kubernetes Secret for the shared KEK, injects it into gateway pods, and
+stores encrypted credential envelopes in the OpenShell database. For
+`workload.kind=deployment` or multi-replica gateways, confirm
+`server.externalDbSecret` points at a shared database. A render/install error
+mentioning `server.credentialDrivers` means the values selected multiple
+external credential backends.
+
 For HA or PostgreSQL-backed installs, also check the external database Secret
 referenced by `server.externalDbSecret` and the PostgreSQL workload if the test
 or operator deployed one in-cluster:

@@ -26,6 +26,7 @@ jobs:
       run_core_e2e: ${{ steps.labels.outputs.run_core_e2e }}
       run_gpu_e2e: ${{ steps.labels.outputs.run_gpu_e2e }}
       run_kubernetes_ha_e2e: ${{ steps.labels.outputs.run_kubernetes_ha_e2e }}
+      run_kubernetes_credential_drivers_e2e: ${{ steps.labels.outputs.run_kubernetes_credential_drivers_e2e }}
       run_any_e2e: ${{ steps.labels.outputs.run_any_e2e }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -44,6 +45,7 @@ jobs:
               run_core_e2e="$(jq -r 'index("test:e2e") != null' <<< "$LABELS_JSON")"
               run_gpu_e2e="$(jq -r 'index("test:e2e-gpu") != null' <<< "$LABELS_JSON")"
               run_kubernetes_ha_e2e="$(jq -r 'index("test:e2e-kubernetes") != null' <<< "$LABELS_JSON")"
+              run_kubernetes_credential_drivers_e2e="$(jq -r 'index("test:e2e-kubernetes") != null' <<< "$LABELS_JSON")"
               ;;
             merge_group)
               # Merge groups have no PR labels. When GPU E2E is required as documented
@@ -52,14 +54,16 @@ jobs:
               run_core_e2e=true
               run_gpu_e2e=true
               run_kubernetes_ha_e2e=false
+              run_kubernetes_credential_drivers_e2e=false
               ;;
             *)
               run_core_e2e=true
               run_gpu_e2e=true
               run_kubernetes_ha_e2e=true
+              run_kubernetes_credential_drivers_e2e=true
               ;;
           esac
-          if [ "$run_core_e2e" = "true" ] || [ "$run_gpu_e2e" = "true" ] || [ "$run_kubernetes_ha_e2e" = "true" ]; then
+          if [ "$run_core_e2e" = "true" ] || [ "$run_gpu_e2e" = "true" ] || [ "$run_kubernetes_ha_e2e" = "true" ] || [ "$run_kubernetes_credential_drivers_e2e" = "true" ]; then
             run_any_e2e=true
           else
             run_any_e2e=false
@@ -68,6 +72,7 @@ jobs:
             echo "run_core_e2e=$run_core_e2e"
             echo "run_gpu_e2e=$run_gpu_e2e"
             echo "run_kubernetes_ha_e2e=$run_kubernetes_ha_e2e"
+            echo "run_kubernetes_credential_drivers_e2e=$run_kubernetes_credential_drivers_e2e"
             echo "run_any_e2e=$run_any_e2e"
           } >> "$GITHUB_OUTPUT"
 
@@ -192,6 +197,18 @@ jobs:
       external-postgres-secret: openshell-ha-pg
       cli-artifact-prefix: rust-binary-cli
 
+  kubernetes-credential-drivers-e2e:
+    needs: [pr_metadata, build-gateway, build-supervisor]
+    if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true'
+    permissions:
+      contents: read
+      packages: read
+    uses: ./.github/workflows/e2e-kubernetes-test.yml
+    with:
+      image-tag: ${{ github.sha }}
+      job-name: Kubernetes Credential Drivers E2E
+      e2e-task: e2e:kubernetes:credential-drivers
+
   core-e2e-result:
     name: Core E2E result
     needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e]
@@ -282,3 +299,30 @@ jobs:
             fi
           done
           exit "$failed"
+
+  kubernetes-credential-drivers-e2e-result:
+    name: Kubernetes Credential Drivers E2E result
+    needs: [pr_metadata, build-gateway, build-supervisor, kubernetes-credential-drivers-e2e]
+    if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Verify Kubernetes credential drivers E2E jobs
+        env:
+          BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }}
+          BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }}
+          KUBERNETES_CREDENTIAL_DRIVERS_E2E_RESULT: ${{ needs.kubernetes-credential-drivers-e2e.result }}
+        run: |
+          set -euo pipefail
+          failed=0
+          for item in \
+            "build-gateway:$BUILD_GATEWAY_RESULT" \
+            "build-supervisor:$BUILD_SUPERVISOR_RESULT" \
+            "kubernetes-credential-drivers-e2e:$KUBERNETES_CREDENTIAL_DRIVERS_E2E_RESULT"; do
+            name="${item%%:*}"
+            result="${item#*:}"
+            if [ "$result" != "success" ]; then
+              echo "::error::$name concluded $result"
+              failed=1
+            fi
+          done
+          exit "$failed"
@@ -37,6 +37,11 @@ on:
         required: false
         type: string
         default: "v0.5.0"
+      e2e-task:
+        description: "mise task to run for the Kubernetes e2e job"
+        required: false
+        type: string
+        default: "e2e:kubernetes"
       mise-version:
         description: "mise version to install on the bare Kubernetes e2e runner"
         required: false
@@ -130,12 +135,13 @@ jobs:
             kind load image-archive "$archive" --name "$KIND_CLUSTER_NAME"
           done
 
-      - name: Run Kubernetes E2E (Rust smoke)
+      - name: Run Kubernetes E2E
         env:
           AGENT_SANDBOX_VERSION: ${{ inputs.agent-sandbox-version }}
           OPENSHELL_E2E_KUBE_CONTEXT: kind-${{ env.KIND_CLUSTER_NAME }}
           OPENSHELL_E2E_KUBE_EXTRA_VALUES: ${{ inputs.extra-helm-values }}
           OPENSHELL_E2E_KUBE_EXTERNAL_POSTGRES_SECRET: ${{ inputs.external-postgres-secret }}
           IMAGE_TAG: ${{ inputs.image-tag }}
           OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell
-        run: mise run --no-deps --skip-deps e2e:kubernetes
+          E2E_TASK: ${{ inputs.e2e-task }}
+        run: mise run --no-deps --skip-deps "$E2E_TASK"
@@ -51,7 +51,7 @@ jobs:
               status_summary="The matching required CI gate status on this PR will flip green automatically once the run finishes."
               ;;
             test:e2e-kubernetes)
-              suite_summary="Kubernetes HA E2E"
+              suite_summary="Kubernetes HA and credential-driver E2E"
               build_summary="gateway and supervisor images"
               status_summary="This is an optional proof-of-life suite; failures are visible in the workflow run but do not publish a required CI gate status."
               ;;

@@ -41,6 +41,8 @@ These pipelines connect skills into end-to-end workflows. Individual skill files
 | `crates/openshell-sdk/` | Shared client SDK | Async Rust gateway client (gRPC transport, TLS, OIDC refresh, edge tunnel); consumed by CLI, TUI, and `@openshell/sdk` |
 | `crates/openshell-providers/` | Provider management | Credential provider backends |
 | `crates/openshell-tui/` | Terminal UI | Ratatui-based dashboard for monitoring |
+| `crates/openshell-driver-kubernetes-secrets/` | Kubernetes Secrets credential driver | In-process `CredentialDriver` backend for OpenShell-managed K8s Secret storage |
+| `crates/openshell-driver-vault/` | Vault credential driver | In-process `CredentialDriver` backend for Vault-compatible KV storage |
 | `crates/openshell-driver-kubernetes/` | Kubernetes compute driver | In-process `ComputeDriver` backend for K8s sandbox pods |
 | `crates/openshell-driver-docker/` | Docker compute driver | In-process `ComputeDriver` backend for local Docker sandbox containers |
 | `crates/openshell-driver-podman/` | Podman compute driver | In-process `ComputeDriver` backend for local Podman sandbox containers |

@@ -18,10 +18,11 @@ Three opt-in labels enable the long-running E2E suites:
   suites in `Branch E2E Checks`
 - `test:e2e-gpu` runs GPU E2E in `Branch E2E Checks`
 - `test:e2e-kubernetes` runs Kubernetes E2E with the HA Helm overlay
-  (`replicaCount: 2` and bundled PostgreSQL) in `Branch E2E Checks`
+  (`replicaCount: 2` and bundled PostgreSQL) and the credential-driver suite
+  (Kubernetes Secrets plus Vault) in `Branch E2E Checks`
 
 When multiple labels are present, `Branch E2E Checks` builds the shared gateway and supervisor images once, builds one CLI artifact per runner architecture, builds the Linux VM driver artifact once, and fans out all enabled suites in parallel. Docker, Podman, GPU, Rust, Python, MCP, and VM E2E jobs reuse the matching prebuilt gateway and CLI binaries instead of compiling additional debug binaries in each job; Kubernetes E2E consumes the gateway image directly and reuses the prebuilt CLI. VM E2E also reuses the prebuilt VM driver artifact and falls back to local VM-driver/runtime preparation for local runs or workflow invocations that omit the artifact.
-The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` is optional while HA behavior is under active iteration: failures are visible in the workflow run but do not publish a required CI gate status.
+The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` is optional while Kubernetes HA and credential-driver behavior are under active iteration: failures are visible in the workflow run but do not publish a required CI gate status.
 
 The GitHub ruleset should require the `OpenShell / ...` statuses published by `Required CI Gates`, not the push-triggered workflow jobs directly.
 
@@ -135,7 +136,7 @@ The bot's full administrator documentation is internal to NVIDIA. The only comma
 | File | Role |
 |---|---|
 | `.github/workflows/branch-checks.yml` | Required non-E2E checks. Triggers on `push: pull-request/[0-9]+` for PR mirrors and `merge_group` for queued merges. |
-| `.github/workflows/branch-e2e.yml` | Standard, GPU, and Kubernetes HA E2E. PR mirror pushes use `test:e2e`, `test:e2e-gpu`, and `test:e2e-kubernetes` labels; merge groups run core and GPU E2E. |
+| `.github/workflows/branch-e2e.yml` | Standard, GPU, Kubernetes HA, and Kubernetes credential-driver E2E. PR mirror pushes use `test:e2e`, `test:e2e-gpu`, and `test:e2e-kubernetes` labels; merge groups run core and GPU E2E. |
 | `.github/workflows/helm-lint.yml` | Helm chart validation. PR mirror pushes skip lint jobs unless Helm inputs changed; merge groups always validate Helm because they represent the final integration state. |
 | `.github/actions/pr-gate/action.yml` | Composite action that resolves PR metadata and verifies the required label is set for PR mirror pushes. Non-push events are allowed through. |
 | `.github/actions/pr-merge-base/action.yml` | Composite action that resolves and fetches the merge-base commit for `pull-request/<N>` push workflows. |

@@ -95,6 +95,7 @@ sha2 = "0.10"
 rand = "0.9"
 jsonwebtoken = "9"
 getrandom = "0.3"
+ring = "0.17"
 spiffe = { version = "0.15", default-features = false, features = ["workload-api-jwt", "tracing"] }
 
 # Filesystem embedding

@@ -151,6 +151,7 @@ Suites:
 - Docker suite (`--features e2e-docker`) - common suite plus Docker-only coverage such as Dockerfile image builds, Docker preflight checks, and managed Docker gateway resume.
 - Docker GPU suite (`--features e2e-docker-gpu`) - Docker suite plus GPU sandbox smoke coverage.
 - VM suite (`--features e2e-vm`) - runs e2e tests on a VM.
+- Kubernetes credential-driver suite (`--features e2e-kubernetes-credential-drivers`) - targeted Kubernetes Secrets and Vault provider credential storage coverage.
 
 GPU device-selection tests compare OpenShell sandboxes against a plain Docker or
 Podman container that requests `--device nvidia.com/gpu=all`. The probe image
@@ -180,6 +181,14 @@ Run the VM-backed Rust CLI e2e suite:
 mise run e2e:vm
 ```
 
+Run the targeted Kubernetes credential-driver e2e suite. This deploys an
+OpenBao fixture for the Vault-compatible driver path and validates Kubernetes
+Secrets and Vault storage backends one at a time:
+
+```shell
+mise run e2e:kubernetes:credential-drivers
+```
+
 Run a single test directly with cargo:
 
 ```shell
@@ -210,3 +219,4 @@ The harness (`e2e/rust/src/harness/`) provides:
 | `OPENSHELL_GATEWAY` | Override active gateway name for E2E tests |
 | `OPENSHELL_GATEWAY_ENDPOINT` | Run E2E tests against an existing plaintext HTTP gateway endpoint |
 | `OPENSHELL_E2E_DRIVER` | Driver name exported by the e2e gateway wrapper (`docker`, `podman`, or `vm`) |
+| `OPENSHELL_E2E_CREDENTIAL_DRIVERS` | Enables the Kubernetes credential-driver fixture path in `e2e/with-kube-gateway.sh` |
@@ -277,7 +277,13 @@ keeps only the current injectable credential values and optional per-credential
 expiry timestamps. A refresh normally mints one credential, but a strategy may
 co-mint several (AWS STS mints the access key, secret key, and session token in
 one call); the refresh state pins the resolved set of env keys it owns so
-collision checks reserve all of them before the first mint.
+collision checks reserve all of them before the first mint. Provider records
+keep inline credential values only for legacy records created before credential
+driver storage. New provider writes keep driver-owned credential handles. When
+no external credential driver is configured, gateways use server-owned encrypted
+database credential storage for defense in depth. Multi-replica deployments can
+use that default with a shared database and shared key-encryption key, or opt
+into an external backend such as Vault or Kubernetes Secrets.
 
 ### Optimistic Concurrency (CAS)