feat!: volume-based TLS identities

core/Dockerfile

@@ -13,7 +13,7 @@ WORKDIR /workspace
 # Clone and build Kopia with cache mounts and cross-compilation
 RUN --mount=type=cache,target=/go/pkg/mod,id=kopia-mod-${TARGETARCH} \
     --mount=type=cache,target=/root/.cache/go-build,id=kopia-build-${TARGETARCH} \
-    git clone --depth=1 --branch=klio https://github.com/cloudnative-pg/kopia && \
+    git clone --depth=1 --branch=cert-reload https://github.com/fcanovai/kopia && \
     cd kopia && \
     CGO_ENABLED=0 GOOS="${TARGETOS}" GOARCH="${TARGETARCH}" \
     go build -o kopia main.go

core/internal/client/klioclient/grpcclient/connection.go

@@ -74,8 +74,23 @@ func (c *Connection) StoreWALStreaming(
 	return g, nil
 }
 
-// Connect opens a connection to a Klio server.
-func Connect(clientConfig *config.ClientConfig, address string) (*Connection, error) {
+// loadClientIdentity loads the client key pair referenced by the client
+// configuration, reading the files fresh on every call. It backs the
+// GetClientCertificate callback, so a rotated client identity is
+// presented on new handshakes without restarting the process.
+func loadClientIdentity(clientConfig *config.ClientConfig) (tls.Certificate, error) {
+	clientCertificate, err := tls.LoadX509KeyPair(clientConfig.Wal.ClientCertPath, clientConfig.Wal.ClientKeyPath)
+	if err != nil {
+		return tls.Certificate{}, fmt.Errorf("while parsing the client certificate: %w", err)
+	}
+
+	return clientCertificate, nil
+}
+
+// buildTLSConfig builds the client TLS configuration, reading the server
+// trust bundle once and wiring the client identity for per-handshake
+// reloads. The eager identity load fails fast on invalid files.
+func buildTLSConfig(clientConfig *config.ClientConfig) (*tls.Config, error) {
 	certPEMBlock, err := os.ReadFile(clientConfig.Wal.ServerCertPath)
 	if err != nil {
 		return nil, fmt.Errorf("while reading the server certificate: %w", err)
@@ -86,17 +101,35 @@ func Connect(clientConfig *config.ClientConfig, address string) (*Connection, er
 		return nil, ErrInconsistentCertificate
 	}
 
-	clientCertificate, err := tls.LoadX509KeyPair(clientConfig.Wal.ClientCertPath, clientConfig.Wal.ClientKeyPath)
-	if err != nil {
-		return nil, fmt.Errorf("while parsing the client certificate: %w", err)
+	// Load once to fail fast on invalid files at connection setup.
+	// Afterwards the identity is re-read on every handshake, so a
+	// rotated client certificate takes effect on reconnects without
+	// restarting the process. A reload failure fails that handshake
+	// closed. The server trust bundle (RootCAs) is still read once
+	// here; its rotation relies on process restart.
+	if _, err := loadClientIdentity(clientConfig); err != nil {
+		return nil, err
 	}
 
-	tlsConfig := &tls.Config{
+	return &tls.Config{
 		RootCAs:    serverCertificatePool,
 		MinVersion: tls.VersionTLS12,
-		Certificates: []tls.Certificate{
-			clientCertificate,
+		GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			clientCertificate, err := loadClientIdentity(clientConfig)
+			if err != nil {
+				return nil, err
+			}
+
+			return &clientCertificate, nil
 		},
+	}, nil
+}
+
+// Connect opens a connection to a Klio server.
+func Connect(clientConfig *config.ClientConfig, address string) (*Connection, error) {
+	tlsConfig, err := buildTLSConfig(clientConfig)
+	if err != nil {
+		return nil, err
 	}
 
 	conn, err := grpc.NewClient(

core/internal/client/klioclient/kopia/kopia.go

@@ -119,12 +119,6 @@ func connectToKopiaServer(
 		return nil, fmt.Errorf("while writing a temporary Kopia config: %w", err)
 	}
 
-	certificateFingerprint, err := kopia.ExtractSHA256CertificateFingerprint(
-		clientConfig.Base.ServerCertPath)
-	if err != nil {
-		return nil, fmt.Errorf("error while extracting fingerprint of the kopia server certificate: %w", err)
-	}
-
 	clientCertificate, err := tls.LoadX509KeyPair(
 		clientConfig.Base.ClientCertPath,
 		clientConfig.Base.ClientKeyPath,
@@ -179,12 +173,12 @@ func connectToKopiaServer(
 			CacheDirectory:     cacheDirectory,
 			ReadOnly:           readOnly,
 		},
-		URL:                   kopiaURL,
-		ClientCertPath:        clientConfig.Base.ClientCertPath,
-		ClientKeyPath:         clientConfig.Base.ClientKeyPath,
-		ServerCertFingerprint: certificateFingerprint,
-		Username:              userName,
-		Hostname:              certHostName,
+		URL:              kopiaURL,
+		ClientCertPath:   clientConfig.Base.ClientCertPath,
+		ClientKeyPath:    clientConfig.Base.ClientKeyPath,
+		ServerCertCAFile: clientConfig.Base.ServerCertPath,
+		Username:         userName,
+		Hostname:         certHostName,
 	}); err != nil {
 		return nil, fmt.Errorf("while executing Kopia command: %w", err)
 	}

core/internal/consumer/backup.go

@@ -75,8 +75,9 @@ type BackupOptions struct {
 	// Tier1ServerAddress is the address of the tier 1 Kopia server.
 	Tier1ServerAddress string
 
-	// Tier1ServerCertificateFingerprint is the SHA256 fingerprint of the tier 1 server certificate.
-	Tier1ServerCertificateFingerprint string
+	// ServerTLSCertFile is the path to the PEM file with the serving
+	// certificate shared by the tier 1 and tier 2 Kopia servers.
+	ServerTLSCertFile string
 
 	// A config file to connect to tier 2
 	Tier2KopiaConfig string
@@ -93,9 +94,6 @@ type BackupOptions struct {
 	// Tier2ServerAddress is the address of the tier 2 Kopia server.
 	Tier2ServerAddress string
 
-	// Tier2ServerCertificateFingerprint is the SHA256 fingerprint of the tier 2 server certificate.
-	Tier2ServerCertificateFingerprint string
-
 	// Tier2WALRepository is the connection to the tier 2 WAL repository.
 	// Used to apply WAL retention after backup retention is applied.
 	Tier2WALRepository *repository.Connection
@@ -336,10 +334,15 @@ func (d *Backup) maintainTier2(ctx context.Context, task *queue.BackupTask, entr
 }
 
 func (d *Backup) refreshTier1KopiaServer(ctx context.Context) error {
+	fingerprint, err := kopia.LeafFingerprint(d.opts.ServerTLSCertFile)
+	if err != nil {
+		return fmt.Errorf("while fingerprinting the tier 1 server certificate: %w", err)
+	}
+
 	return d.tier1Kopia.RefreshServer(ctx, kopia.RefreshServerOptions{
 		ServerControlUser:     d.opts.RunID,
 		ServerControlPassword: d.opts.RunSecret,
-		ServerCertFingerprint: d.opts.Tier1ServerCertificateFingerprint,
+		ServerCertFingerprint: fingerprint,
 		Address:               d.opts.Tier1ServerAddress,
 	})
 }
@@ -387,10 +390,15 @@ func getPinnedSnapshots(manifests []kopia.Manifest) []string {
 // refreshTier2KopiaServer makes sure the tier 2 kopia server
 // has downloaded the latest manifests from the object store.
 func (d *Backup) refreshTier2KopiaServer(ctx context.Context) error {
+	fingerprint, err := kopia.LeafFingerprint(d.opts.ServerTLSCertFile)
+	if err != nil {
+		return fmt.Errorf("while fingerprinting the tier 2 server certificate: %w", err)
+	}
+
 	return d.tier2Kopia.RefreshServer(ctx, kopia.RefreshServerOptions{
 		ServerControlUser:     d.opts.RunID,
 		ServerControlPassword: d.opts.RunSecret,
-		ServerCertFingerprint: d.opts.Tier2ServerCertificateFingerprint,
+		ServerCertFingerprint: fingerprint,
 		Address:               d.opts.Tier2ServerAddress,
 	})
 }

core/internal/kopia/fingerprint.go

@@ -0,0 +1,58 @@
+/*
+Copyright © contributors to CloudNativePG, established as
+CloudNativePG a Series of LF Projects, LLC.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package kopia
+
+import (
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"os"
+)
+
+// ErrNoCertificateFound is returned when a file contains no PEM-encoded
+// certificate.
+var ErrNoCertificateFound = errors.New("no certificate found in file")
+
+// LeafFingerprint computes the lowercase hex SHA256 fingerprint of the
+// first certificate in a PEM file, matching the format Kopia expects
+// for server certificate pinning.
+func LeafFingerprint(certPath string) (string, error) {
+	pemBytes, err := os.ReadFile(certPath) //nolint:gosec
+	if err != nil {
+		return "", fmt.Errorf("while reading certificate file: %w", err)
+	}
+
+	block, _ := pem.Decode(pemBytes)
+	if block == nil || block.Type != "CERTIFICATE" {
+		return "", fmt.Errorf("%w: %s", ErrNoCertificateFound, certPath)
+	}
+
+	cert, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		return "", fmt.Errorf("while parsing certificate file: %w", err)
+	}
+
+	fingerprint := sha256.Sum256(cert.Raw)
+
+	return hex.EncodeToString(fingerprint[:]), nil
+}

core/internal/kopia/fingerprint_test.go

@@ -0,0 +1,96 @@
+/*
+Copyright © contributors to CloudNativePG, established as
+CloudNativePG a Series of LF Projects, LLC.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package kopia
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func writeFingerprintTestCert(t *testing.T, dir, name string) string {
+	t.Helper()
+
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+
+	template := &x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject:      pkix.Name{CommonName: name},
+		DNSNames:     []string{name},
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}
+
+	der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+	require.NoError(t, err)
+
+	certPath := filepath.Join(dir, name+".crt")
+	require.NoError(t, os.WriteFile(
+		certPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600))
+
+	return certPath
+}
+
+func TestLeafFingerprintTracksRotation(t *testing.T) {
+	dir := t.TempDir()
+	certPath := writeFingerprintTestCert(t, dir, "server")
+
+	first, err := LeafFingerprint(certPath)
+	require.NoError(t, err)
+	require.Len(t, first, 2*sha256.Size)
+
+	// Rewriting the file (a renewal) changes the fingerprint, and a
+	// fresh read picks it up with no restart involved.
+	_ = writeFingerprintTestCert(t, dir, "server")
+
+	second, err := LeafFingerprint(certPath)
+	require.NoError(t, err)
+	require.NotEqual(t, first, second)
+}
+
+func TestLeafFingerprintFailures(t *testing.T) {
+	t.Run("missing file", func(t *testing.T) {
+		_, err := LeafFingerprint(filepath.Join(t.TempDir(), "absent.crt"))
+		require.Error(t, err)
+	})
+
+	t.Run("not a certificate", func(t *testing.T) {
+		dir := t.TempDir()
+		bundlePath := filepath.Join(dir, "ca.crt")
+		require.NoError(t, os.WriteFile(bundlePath, []byte("not a bundle"), 0o600))
+
+		_, err := LeafFingerprint(bundlePath)
+		require.ErrorIs(t, err, ErrNoCertificateFound)
+	})
+}