diff --git a/crates/openshell-server/src/certgen.rs b/crates/openshell-server/src/certgen.rs index 1f3ffca02d..01c920242a 100644 --- a/crates/openshell-server/src/certgen.rs +++ b/crates/openshell-server/src/certgen.rs @@ -24,7 +24,7 @@ use clap::Args; use k8s_openapi::ByteString; -use k8s_openapi::api::core::v1::Secret; +use k8s_openapi::api::core::v1::{ConfigMap, Secret}; use kube::Client; use kube::api::{Api, ObjectMeta, PostParams}; use miette::{IntoDiagnostic, Result, WrapErr}; @@ -78,6 +78,20 @@ pub struct CertgenArgs { /// For local debugging. #[arg(long)] dry_run: bool, + + /// Name of a ConfigMap to create containing the CA certificate (key: ca.crt) + /// for BackendTLSPolicy backend validation. In full PKI mode, the CA comes + /// from the generated bundle. In --jwt-only mode, the CA is read from + /// --backend-ca-source-secret. + #[arg(long, value_name = "NAME")] + backend_ca_configmap_name: Option, + + /// Name of an existing Secret containing a ca.crt key to populate the + /// backend CA ConfigMap from. Required with --jwt-only when + /// --backend-ca-configmap-name is set (typically the server TLS Secret + /// created by cert-manager). + #[arg(long, value_name = "NAME", requires = "backend_ca_configmap_name")] + backend_ca_source_secret: Option, } pub async fn run(args: CertgenArgs) -> Result<()> { @@ -97,7 +111,13 @@ pub async fn run(args: CertgenArgs) -> Result<()> { run_local(dir, &args.server_sans) } else { let bundle = generate_pki(&args.server_sans)?; - run_kubernetes(&args, &bundle).await + run_kubernetes(&args, &bundle).await?; + + if let Some(ref cm_name) = args.backend_ca_configmap_name { + create_backend_ca_configmap_if_needed(&args, &bundle, cm_name).await?; + } + + Ok(()) } } @@ -293,6 +313,97 @@ async fn create_tls_secrets( Ok(()) } +async fn create_backend_ca_configmap_if_needed( + args: &CertgenArgs, + bundle: &PkiBundle, + configmap_name: &str, +) -> Result<()> { + let namespace = args + .namespace + .as_deref() + .ok_or_else(|| miette::miette!("--namespace is required (or set POD_NAMESPACE)"))?; + + let client = Client::try_default() + .await + .into_diagnostic() + .wrap_err("failed to construct Kubernetes client for backend CA ConfigMap")?; + let api: Api = Api::namespaced(client.clone(), namespace); + + if api + .get_opt(configmap_name) + .await + .into_diagnostic() + .wrap_err_with(|| format!("failed to read configmap {configmap_name}"))? + .is_some() + { + info!( + namespace = %namespace, + configmap = %configmap_name, + "Backend CA ConfigMap already exists, skipping." + ); + return Ok(()); + } + + let ca_pem = if !args.jwt_only { + bundle.ca_cert_pem.clone() + } else if let Some(source_secret) = &args.backend_ca_source_secret { + let secret_api: Api = Api::namespaced(client, namespace); + match secret_api + .get_opt(source_secret) + .await + .into_diagnostic() + .wrap_err_with(|| format!("failed to read secret {source_secret}"))? + { + Some(secret) => { + let data = secret.data.ok_or_else(|| { + miette::miette!("secret {source_secret} has no data") + })?; + let ca = data.get("ca.crt").ok_or_else(|| { + miette::miette!("secret {source_secret} has no ca.crt key") + })?; + String::from_utf8(ca.0.clone()) + .into_diagnostic() + .wrap_err("ca.crt is not valid UTF-8")? + } + None => { + warn!( + secret = %source_secret, + configmap = %configmap_name, + "Backend CA source secret not found; ConfigMap not created. \ + Create it manually or run helm upgrade after the TLS secret exists." + ); + return Ok(()); + } + } + } else { + return Err(miette::miette!( + "--backend-ca-source-secret is required with --jwt-only \ + and --backend-ca-configmap-name" + )); + }; + + let configmap = ConfigMap { + metadata: ObjectMeta { + name: Some(configmap_name.to_string()), + ..Default::default() + }, + data: Some(BTreeMap::from([("ca.crt".to_string(), ca_pem)])), + ..Default::default() + }; + + api.create(&PostParams::default(), &configmap) + .await + .into_diagnostic() + .wrap_err_with(|| format!("failed to create configmap {configmap_name}"))?; + + info!( + namespace = %namespace, + configmap = %configmap_name, + "Backend CA ConfigMap created." + ); + Ok(()) +} + fn tls_secret(name: &str, crt_pem: &str, key_pem: &str, ca_pem: &str) -> Secret { let mut data = BTreeMap::new(); data.insert( diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 2e86c3a1b5..f2e623d3e1 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -1138,6 +1138,59 @@ mod tests { )); } + #[test] + fn generate_certs_backend_ca_configmap_flags_parse() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _g1 = EnvVarGuard::remove("OPENSHELL_DB_URL"); + let _g2 = EnvVarGuard::remove("POD_NAMESPACE"); + + let cli = Cli::try_parse_from([ + "openshell-gateway", + "generate-certs", + "--namespace", + "openshell", + "--jwt-only", + "--jwt-secret-name", + "openshell-jwt-keys", + "--backend-ca-configmap-name", + "openshell-backend-ca", + "--backend-ca-source-secret", + "openshell-server-tls", + ]) + .expect("backend CA ConfigMap flags should parse with --jwt-only"); + + assert!(matches!( + cli.command, + Some(super::Commands::GenerateCerts(_)) + )); + } + + #[test] + fn generate_certs_backend_ca_source_secret_requires_configmap_name() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _g1 = EnvVarGuard::remove("OPENSHELL_DB_URL"); + let _g2 = EnvVarGuard::remove("POD_NAMESPACE"); + + let err = Cli::try_parse_from([ + "openshell-gateway", + "generate-certs", + "--namespace", + "openshell", + "--jwt-only", + "--jwt-secret-name", + "openshell-jwt-keys", + "--backend-ca-source-secret", + "openshell-server-tls", + ]) + .expect_err("--backend-ca-source-secret should require --backend-ca-configmap-name"); + + assert_eq!(err.kind(), clap::error::ErrorKind::MissingRequiredArgument); + } + #[test] fn bare_invocation_with_no_db_url_parses_for_runtime_defaults() { // db_url is Option at the clap level so subcommand parsing diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 93dab354b6..426c5aba7e 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -41,6 +41,9 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -backend-ca when empty. The certgen hook auto-creates this: with pkiInitJob (default), immediately on install/upgrade; with cert-manager, the hook waits for cert-manager to issue the server certificate first, so on first install the ConfigMap is created during the subsequent `helm upgrade` (run upgrade after cert-manager reconciles). | +| grpcRoute.backendTLSPolicy.enabled | bool | `false` | Create a BackendTLSPolicy resource for end-to-end TLS between the Gateway proxy and the OpenShell gateway pod. The traffic flow is: client → HTTPS → Gateway (terminate) → TLS (re-encrypt) → gateway pod. Requires server.disableTls=false and server.tls.enableMtls=false. The certgen hook auto-creates the backend CA ConfigMap. | +| grpcRoute.backendTLSPolicy.hostname | string | `""` | Hostname the Gateway proxy validates against the backend's TLS certificate SAN. Defaults to the service FQDN (..svc.cluster.local) when empty, which matches the SAN included by both cert-manager and the pkiInitJob. | | grpcRoute.enabled | bool | `false` | Create a Gateway API GRPCRoute for the gateway service. | | grpcRoute.gateway.className | string | `"eg"` | GatewayClass to reference. Envoy Gateway installs one named "eg". | | grpcRoute.gateway.create | bool | `false` | When true, a Gateway resource is created in the release namespace. Set to false and provide name/namespace to attach to a pre-existing Gateway. | @@ -262,8 +268,9 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | server.sandboxNamespace | string | `""` | Namespace where sandbox pods are created. Defaults to the Helm release namespace (.Release.Namespace) when left empty. | | server.telemetryEnabled | bool | `true` | Enable anonymous OpenShell telemetry from the gateway and the sandbox supervisors it launches. | | server.tls.certSecretName | string | `"openshell-server-tls"` | K8s secret (type kubernetes.io/tls) with tls.crt and tls.key for the server. | -| server.tls.clientCaSecretName | string | `"openshell-server-client-ca"` | K8s secret with ca.crt for client certificate verification (mTLS). Set to "" to disable mTLS and run HTTPS-only (use OIDC for auth instead). | +| server.tls.clientCaSecretName | string | `"openshell-server-client-ca"` | K8s secret with ca.crt for client certificate verification (mTLS). Only used when enableMtls is true. Set to "" to use the default client CA (from pkiInitJob or cert-manager). | | server.tls.clientTlsSecretName | string | `"openshell-client-tls"` | K8s secret mounted into sandbox pods for mTLS to the server. | +| server.tls.enableMtls | bool | `true` | Enable mTLS client certificate authentication. When false, the gateway runs HTTPS-only without requiring client certificates (use OIDC for auth instead). Must be false when using BackendTLSPolicy because ingress proxies cannot present client certificates to the backend. | | server.workspaceDefaultStorageSize | string | `""` | Default storage size for the workspace PVC in sandbox pods. Uses Kubernetes quantity syntax (e.g. "2Gi", "10Gi", "500Mi"). Empty = built-in default (2Gi). | | server.workspaceStorageClass | string | `""` | Kubernetes StorageClass for the workspace PVC in sandbox pods. Empty (default) = omit storageClassName, using the cluster's default StorageClass. Set this on clusters with no default StorageClass, otherwise the workspace PVC stays Pending and the sandbox never starts. | | service.healthPort | int | `8081` | Gateway health service port. | diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index 0242d8118c..d0a528de12 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -41,6 +41,9 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -backend-ca when empty. The certgen hook auto-creates this: + # with pkiInitJob (default), immediately on install/upgrade; with + # cert-manager, the hook waits for cert-manager to issue the server + # certificate first, so on first install the ConfigMap is created during + # the subsequent `helm upgrade` (run upgrade after cert-manager reconciles). + caCertificateConfigMapName: "" + # -- Hostname the Gateway proxy validates against the backend's TLS + # certificate SAN. Defaults to the service FQDN + # (..svc.cluster.local) when empty, which matches + # the SAN included by both cert-manager and the pkiInitJob. + hostname: "" # OpenShift Route with TLS passthrough. The gateway terminates its own # TLS/mTLS; the router only forwards based on SNI, so it never sees plaintext diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index 51284465d9..2b397fc6cf 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -135,6 +135,45 @@ openshell status See [Authentication](/kubernetes/setup) for OIDC issuer, audience, and roles configuration. +## End-to-end TLS (BackendTLSPolicy) + +As an alternative to the plaintext backend path above, the chart can create a `BackendTLSPolicy` that tells the Gateway proxy to re-encrypt traffic when connecting to the OpenShell gateway pod: + +```text +client → HTTPS → Gateway (terminate TLS) → TLS (re-encrypt) → openshell gateway pod +``` + +This keeps TLS on the gateway pod rather than disabling it with `server.disableTls=true`. The Gateway proxy validates the backend's certificate against a CA ConfigMap that the certgen hook auto-creates. + +BackendTLSPolicy is a standard Gateway API resource. It is supported on OpenShift 4.22+ (via the OpenShift gateway controller) and on other platforms where the Gateway API implementation supports it (check your controller's documentation). + +### Install with e2e TLS + +The certgen hook automatically creates the backend CA ConfigMap when `backendTLSPolicy` is enabled: + +```shell +helm upgrade --install openshell \ + oci://ghcr.io/nvidia/openshell/helm-chart \ + --version \ + --namespace openshell \ + --set server.tls.enableMtls=false \ + --set grpcRoute.enabled=true \ + --set grpcRoute.gateway.create=true \ + --set grpcRoute.gateway.className=eg \ + --set grpcRoute.gateway.listener.protocol=HTTPS \ + --set grpcRoute.gateway.listener.port=443 \ + --set 'grpcRoute.gateway.listener.tls.certificateRefs[0].name=openshell-ingress-tls' \ + --set grpcRoute.backendTLSPolicy.enabled=true \ + --set server.oidc.issuer=https:// \ + --set 'grpcRoute.hostnames[0]=' +``` + +Note that `server.disableTls` is **not** set — the gateway pod continues to serve TLS — but `server.tls.enableMtls=false` disables mTLS client certificate authentication because the Gateway proxy cannot present a client certificate to the backend. The BackendTLSPolicy hostname defaults to the service FQDN, which matches the SAN on the server certificate. Use OIDC for authentication (configured via `server.oidc.issuer`). + +The example above uses the default `pkiInitJob` for TLS, which creates the backend CA ConfigMap immediately. If using cert-manager instead (`--set certManager.enabled=true`), you'll need to run `helm upgrade` a second time after cert-manager issues the server certificate so the certgen hook can create the backend CA ConfigMap from it. + +For OpenShift 4.22+, see [OpenShift](/kubernetes/openshift#end-to-end-tls-openshift-422) for platform-specific instructions including Gateway and GatewayClass setup. + ## SSH Relay Sandbox SSH uses the gateway endpoint registered with the CLI. No separate Helm SSH host or port values are required. diff --git a/docs/kubernetes/managing-certificates.mdx b/docs/kubernetes/managing-certificates.mdx index c4cb07f57e..d97c1e24be 100644 --- a/docs/kubernetes/managing-certificates.mdx +++ b/docs/kubernetes/managing-certificates.mdx @@ -66,7 +66,7 @@ By default, cert-manager issues both the server and client certificates from a self-signed CA the chart creates — this rotates automatically, but the server certificate is still not publicly trusted. `certManager.serverIssuerRef` overrides the `issuerRef` on the server `Certificate` resource to point at a -real `Issuer` or `ClusterIssuer` instead, for example an ACME issuer: +real `Issuer` or `ClusterIssuer` instead, for example a LetsEncrypt/ACME issuer: ```shell helm upgrade --install openshell \ diff --git a/docs/kubernetes/openshift.mdx b/docs/kubernetes/openshift.mdx index 43e7d0338b..d38347c8a7 100644 --- a/docs/kubernetes/openshift.mdx +++ b/docs/kubernetes/openshift.mdx @@ -87,13 +87,121 @@ openshell gateway add http://127.0.0.1:8080 --local --name openshift openshell status ``` -## Production: expose externally with a real certificate +## Options for end-to-end TLS -The steps above run the gateway over plaintext HTTP for quick evaluation. For -a real deployment, cert-manager can issue the gateway's server certificate -from a real Issuer or ClusterIssuer (for example, an ACME issuer), and an -OpenShift Route with TLS passthrough exposes it externally while the gateway -keeps terminating its own TLS and mTLS. +The steps above run the gateway over plaintext HTTP for quick evaluation. For production deployments, choose one of the approaches below based on your OpenShift version and preferences. + +### End-to-end TLS using Gateway API and BackendTLSPolicy (OpenShift 4.22+) + +OpenShift 4.22 and later support `BackendTLSPolicy` in the Gateway API, enabling end-to-end TLS between the OpenShift router and the OpenShell gateway pod. The traffic flow is: + +```text +client → HTTPS → OpenShift Gateway (terminate TLS) → TLS (re-encrypt) → openshell gateway pod +``` + +This removes the requirement to run the gateway with `server.disableTls=true`. The OpenShift router terminates client-facing TLS at the listener and re-encrypts when connecting to the backend service, validating the backend's certificate against a CA you provide. + +#### Prerequisites + +- OpenShift 4.22+ cluster with the Gateway API enabled +- cert-manager installed (recommended) or the built-in pkiInitJob for server certificates +- A `GatewayClass` registered for the OpenShift gateway controller + +#### Create the GatewayClass + +If your cluster does not already have an OpenShift GatewayClass, create one: + +```shell +oc apply -f - <<'EOF' +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: openshift-default +spec: + controllerName: openshift.io/gateway-controller/v1 +EOF +``` + +#### Create the Gateway + +Create a Gateway resource in the `openshift-ingress` namespace. Replace `` with your cluster's route hostname (typically a wildcard like `*.openshell-ingress-gw.example.com`): + +```shell +oc apply -f - <<'EOF' +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: openshell-gateway + namespace: openshift-ingress +spec: + gatewayClassName: openshift-default + listeners: + - name: grpc + hostname: "" + port: 443 + protocol: HTTPS + tls: + mode: Terminate + certificateRefs: + - name: + kind: Secret + allowedRoutes: + namespaces: + from: Selector + selector: + matchLabels: + kubernetes.io/metadata.name: openshell +EOF +``` + +The listener TLS Secret should contain the certificate for the external hostname. + +#### Install with e2e TLS + +Install the chart with the GRPCRoute and BackendTLSPolicy enabled. The certgen hook automatically creates the backend CA ConfigMap from the generated PKI bundle: + +```shell +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ + --version \ + --namespace openshell \ + --set podSecurityContext.fsGroup=null \ + --set securityContext.runAsUser=null \ + --set server.tls.enableMtls=false \ + --set grpcRoute.enabled=true \ + --set grpcRoute.gateway.name=openshell-gateway \ + --set grpcRoute.gateway.namespace=openshift-ingress \ + --set 'grpcRoute.hostnames[0]=' \ + --set grpcRoute.backendTLSPolicy.enabled=true +``` + +| Override | Reason | +|---|---| +| `podSecurityContext.fsGroup=null` / `securityContext.runAsUser=null` | Let OpenShift's SCC admission assign UIDs. | +| `server.tls.enableMtls=false` | Disable mTLS client certificate authentication. BackendTLSPolicy only validates the server certificate; the ingress proxy cannot present a client certificate to the backend. Use OIDC for authentication instead. | +| `grpcRoute.enabled=true` | Create a GRPCRoute pointing at the external Gateway. | +| `grpcRoute.gateway.name` / `namespace` | Reference the Gateway created above in `openshift-ingress`. | +| `grpcRoute.backendTLSPolicy.enabled=true` | Create a BackendTLSPolicy for TLS re-encryption to the gateway pod. The certgen hook auto-creates the backend CA ConfigMap. The Gateway proxy validates the backend certificate against the service FQDN, which is already in the default server certificate SANs. | +| `grpcRoute.hostnames` | External hostname for the GRPCRoute. This goes on the Gateway listener certificate, not the backend certificate. | + +Note that `server.disableTls` is **not** set — the gateway pod serves TLS over HTTPS without requiring client certificates. Use OIDC for authentication (see [Access Control](/kubernetes/access-control)). + +**Using cert-manager instead of pkiInitJob:** Add `--set certManager.enabled=true` to the install command. The default `certManager.serverDnsNames` already includes the service FQDN needed for BackendTLSPolicy validation. With cert-manager, the backend CA ConfigMap requires a two-step install because cert-manager creates the server certificate *after* the chart is installed: + +1. `helm install` — cert-manager issues the server certificate, but the certgen hook can't create the backend CA ConfigMap yet (logs a warning) +2. `helm upgrade` — certgen hook now reads the CA from the cert-manager-issued server certificate and creates the backend CA ConfigMap + +The gateway won't accept backend connections until step 2 completes. Wait for the cert-manager Certificate to be Ready before running upgrade. + +#### Register over HTTPS + +```shell +openshell gateway add https:// --name openshift +openshell status +``` + +### End-to-end TLS using pass-through Route (all OpenShift versions) + +For OpenShift versions prior to 4.22, or when you prefer Route-based ingress, cert-manager can issue the gateway's server certificate from a real Issuer or ClusterIssuer (for example, a LetsEncrypt/ACME issuer), and an OpenShift Route with TLS passthrough exposes it externally while the gateway keeps terminating its own TLS and mTLS. Install cert-manager and configure a working `ClusterIssuer` first — see [Managing Certificates](/kubernetes/managing-certificates) for the