diff --git a/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/DefaultMtlsProviderFactory.java b/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/DefaultMtlsProviderFactory.java index 8538f10c0d48..7b9b824d5e87 100644 --- a/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/DefaultMtlsProviderFactory.java +++ b/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/DefaultMtlsProviderFactory.java @@ -40,9 +40,10 @@ public class DefaultMtlsProviderFactory { /** * Creates an instance of {@link MtlsProvider}. It first attempts to create an {@link - * com.google.auth.mtls.X509Provider}. If the certificate source is unavailable, it falls back to - * creating a {@link SecureConnectProvider}. If the secure connect provider also fails, it throws - * a {@link com.google.auth.mtls.CertificateSourceUnavailableException}. + * com.google.auth.mtls.X509Provider}, which also falls back to the GKE credential bundle when no + * certificate configuration is present. If the certificate source is unavailable, it falls back + * to creating a {@link SecureConnectProvider}. If the secure connect provider also fails, it + * throws a {@link com.google.auth.mtls.CertificateSourceUnavailableException}. * * @return an instance of {@link MtlsProvider}. * @throws com.google.auth.mtls.CertificateSourceUnavailableException if neither provider can be @@ -53,7 +54,7 @@ public static MtlsProvider create() throws IOException { // Note: The caller should handle CertificateSourceUnavailableException gracefully, since // it is an expected error case. All other IOExceptions are unexpected and should be surfaced // up the call stack. - MtlsProvider mtlsProvider = new X509Provider(); + MtlsProvider mtlsProvider = X509Provider.createWithGkeCredentialBundleFallback(); if (mtlsProvider.isAvailable()) { return mtlsProvider; } diff --git a/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/MtlsKeyStoreUtils.java b/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/MtlsKeyStoreUtils.java new file mode 100644 index 000000000000..d8f51c61935f --- /dev/null +++ b/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/MtlsKeyStoreUtils.java @@ -0,0 +1,173 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.auth.mtls; + +import com.google.api.client.util.PemReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.security.spec.PKCS8EncodedKeySpec; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.jspecify.annotations.NullMarked; + +/** + * Builds the mTLS {@link KeyStore} from PEM-encoded certificates and a private key. + * + *

This replaces {@code com.google.api.client.util.SecurityUtils#createMtlsKeyStore}, which keeps + * only the first certificate. Here the key entry holds every distinct certificate in the input, so + * the intermediate certificates are presented too. Some peers (for example, another workload that + * trusts only a root CA) need them to verify the client certificate. Because every certificate in + * the input is sent in the TLS handshake, the input should hold only the leaf certificate and its + * intermediates. + * + *

Everything else matches the upstream method: key store type (JKS), alias, empty password, + * accepted key format (PKCS#8 {@code PRIVATE KEY}), which private key is chosen, and error + * messages. Any input that the upstream method accepted is still accepted, even though more of it + * is now used: once a certificate and a private key have been found (the point where the upstream + * method stopped reading), a malformed later PEM section ends the input instead of failing, and an + * extra certificate that cannot be parsed is left out of the chain. Unlike the upstream method, the + * input is decoded as UTF-8 rather than the platform default charset, and it is read to the end. + */ +@NullMarked +final class MtlsKeyStoreUtils { + + private static final Logger LOGGER = Logger.getLogger(MtlsKeyStoreUtils.class.getName()); + + private MtlsKeyStoreUtils() {} + + /** + * Creates a key store holding one private key entry. + * + * @param certAndKey PEM input containing one or more {@code CERTIFICATE} sections and a PKCS#8 + * {@code PRIVATE KEY} section, in any order. The first certificate must be the one that + * matches the private key (the leaf); the remaining distinct certificates are kept in input + * order as the rest of the chain. If there are several private keys, the one used is the same + * as upstream: the last one before the first certificate, or else the first one after it. The + * stream is read to the end and is not closed. + * @throws IllegalArgumentException if the input has no certificate or no private key, or if a PEM + * section before both a certificate and a private key have been read is malformed + * @throws GeneralSecurityException if the first certificate or the private key cannot be parsed, + * or the key store cannot be created + * @throws IOException if the input cannot be read + */ + static KeyStore createMtlsKeyStore(InputStream certAndKey) + throws GeneralSecurityException, IOException { + List certificates = new ArrayList<>(); + byte[] privateKey = null; + PemReader reader = new PemReader(new InputStreamReader(certAndKey, StandardCharsets.UTF_8)); + while (true) { + PemReader.Section section; + try { + section = reader.readNextSection(); + } catch (IllegalArgumentException e) { + if (!certificates.isEmpty() && privateKey != null) { + // Malformed trailing content after a usable certificate and key: stop here, as the + // upstream method (which stopped reading at this point) would have. + break; + } + throw e; + } + if (section == null) { + break; + } + if ("CERTIFICATE".equals(section.getTitle())) { + byte[] certBytes = section.getBase64DecodedBytes(); + if (!containsCertificate(certificates, certBytes)) { + certificates.add(certBytes); + } + } else if ("PRIVATE KEY".equals(section.getTitle()) + && (privateKey == null || certificates.isEmpty())) { + // Same choice as upstream: a later key replaces an earlier one until the first certificate + // has been seen. + privateKey = section.getBase64DecodedBytes(); + } + } + if (certificates.isEmpty()) { + throw new IllegalArgumentException("certificate is missing from certAndKey string"); + } + if (privateKey == null) { + throw new IllegalArgumentException("private key is missing from certAndKey string"); + } + + CertificateFactory certFactory = CertificateFactory.getInstance("X.509"); + List chain = new ArrayList<>(certificates.size()); + chain.add(parseCertificate(certFactory, certificates.get(0))); + for (int i = 1; i < certificates.size(); i++) { + try { + chain.add(parseCertificate(certFactory, certificates.get(i))); + } catch (CertificateException | RuntimeException e) { + // The upstream method never parsed extra certificates; leave this one out rather than + // fail. + LOGGER.log(Level.FINE, "Skipping unparsable extra certificate in mTLS chain", e); + } + } + PrivateKey key = + KeyFactory.getInstance(chain.get(0).getPublicKey().getAlgorithm()) + .generatePrivate(new PKCS8EncodedKeySpec(privateKey)); + + KeyStore keyStore = KeyStore.getInstance("JKS"); + keyStore.load(null); + keyStore.setKeyEntry("alias", key, new char[] {}, chain.toArray(new X509Certificate[0])); + return keyStore; + } + + private static boolean containsCertificate(List certificates, byte[] candidate) { + for (byte[] existing : certificates) { + if (Arrays.equals(existing, candidate)) { + return true; + } + } + return false; + } + + private static X509Certificate parseCertificate(CertificateFactory certFactory, byte[] der) + throws CertificateException { + Certificate cert = certFactory.generateCertificate(new ByteArrayInputStream(der)); + if (cert == null) { + throw new CertificateException("Empty certificate"); + } + return (X509Certificate) cert; + } +} diff --git a/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/MtlsUtils.java b/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/MtlsUtils.java index a5c4c0f86e77..4f0d4c034b6d 100644 --- a/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/MtlsUtils.java +++ b/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/MtlsUtils.java @@ -33,13 +33,19 @@ import com.google.api.core.InternalApi; import com.google.auth.oauth2.EnvironmentProvider; import com.google.auth.oauth2.PropertyProvider; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; +import com.google.common.io.BaseEncoding; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.MessageDigest; import java.util.Locale; import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; /** * Utility class for mTLS related operations. @@ -53,10 +59,257 @@ public class MtlsUtils { static final String WELL_KNOWN_CERTIFICATE_CONFIG_FILE = "certificate_config.json"; static final String CLOUDSDK_CONFIG_DIRECTORY = "gcloud"; + /** + * Well-known path of the combined certificate chain and private key bundle that GKE delivers to + * pods with Agent Identity (Pod Certificates). Used as a fallback client certificate source when + * no certificate configuration file is configured or present. + */ + static final String GKE_CREDENTIAL_BUNDLE_PATH = + "/var/run/secrets/workload-spiffe-credentials/x509.credential-bundle.private-key.pem"; + + // Overridable via setGkeCredentialBundlePathForTesting so unit tests can point to a temp file. + private static volatile String gkeCredentialBundlePath = GKE_CREDENTIAL_BUNDLE_PATH; + private MtlsUtils() { // Prevent instantiation for Utility class } + /** + * Returns {@code true} if a certificate configuration is configured or present: either {@code + * GOOGLE_API_CERTIFICATE_CONFIG} is set, or the well-known gcloud {@code certificate_config.json} + * file exists. When this returns {@code true}, the GKE credential bundle fallback is not used, so + * the certificate used for mTLS and the certificate bound to tokens always come from the same + * source. + */ + public static boolean hasCertificateConfiguration( + EnvironmentProvider envProvider, PropertyProvider propProvider) { + if (!Strings.isNullOrEmpty(envProvider.getEnv(CERTIFICATE_CONFIGURATION_ENV_VARIABLE))) { + return true; + } + try { + return getWellKnownCertificateConfigFile(envProvider, propProvider).exists(); + } catch (IOException e) { + // APPDATA missing on Windows, etc. No well-known configuration file. + return false; + } + } + + /** + * Returns the GKE credential bundle path if no certificate configuration is configured or present + * (see {@link #hasCertificateConfiguration}) and the bundle exists and is readable; otherwise + * returns {@code null}. + */ + static @Nullable String getGkeCredentialBundlePath( + EnvironmentProvider envProvider, PropertyProvider propProvider) { + if (hasCertificateConfiguration(envProvider, propProvider)) { + return null; + } + return findGkeCredentialBundle(); + } + + /** + * Returns {@code true} if the GKE credential bundle at {@code path} is a readable, non-empty + * regular file. + */ + public static boolean isGkeCredentialBundleReady(String path) { + File bundle = new File(path); + return bundle.isFile() && bundle.canRead() && bundle.length() > 0; + } + + /** + * Returns the GKE credential bundle path if it is a readable, non-empty regular file, otherwise + * null. + */ + private static @Nullable String findGkeCredentialBundle() { + String path = gkeCredentialBundlePath; + return isGkeCredentialBundleReady(path) ? path : null; + } + + /** Overrides the GKE credential bundle path for testing; {@code null} restores the default. */ + @VisibleForTesting + static void setGkeCredentialBundlePathForTesting(@Nullable String path) { + gkeCredentialBundlePath = path != null ? path : GKE_CREDENTIAL_BUNDLE_PATH; + } + + private static boolean isUseClientCertificateExplicitlyDisabled( + @Nullable String useClientCertificate) { + return !Strings.isNullOrEmpty(useClientCertificate) + && !"true".equalsIgnoreCase(useClientCertificate); + } + + /** + * Returns if mutual TLS client certificate should be used. Returns true if valid workload + * certificates are configured or if GOOGLE_API_USE_CLIENT_CERTIFICATE is explicitly set to true + * (e.g. for Enterprise Certificate Proxy or custom MtlsProviders), unless explicitly disabled via + * GOOGLE_API_USE_CLIENT_CERTIFICATE (set to a non-empty value other than "true"). + */ + public static boolean useMtlsClientCertificate( + EnvironmentProvider envProvider, PropertyProvider propProvider) { + String rawVal = envProvider.getEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE"); + String useClientCertificate = rawVal != null ? rawVal.trim() : null; + if (isUseClientCertificateExplicitlyDisabled(useClientCertificate)) { + return false; + } + if (getWorkloadCertPath(envProvider, propProvider) != null) { + return true; + } + return "true".equalsIgnoreCase(useClientCertificate); + } + + /** + * Resolves and returns the path to the mutual TLS client certificate, or null if none should be + * used. + * + *

Possible outcomes: + * + *

    + *
  1. Non-null {@link String} (Valid happy path): A valid workload certificate + * configuration was found and both the certificate and private key files exist and are + * readable; or no certificate configuration is set or present and the GKE credential bundle + * ({@code /var/run/secrets/workload-spiffe-credentials/ + * x509.credential-bundle.private-key.pem}) exists and is readable, in which case the bundle + * path is returned. + *
  2. {@link IllegalStateException} (Invalid state - fail closed): An explicit {@code + * GOOGLE_API_CERTIFICATE_CONFIG} path or an existing default well-known certificate + * configuration file is missing, unreadable, malformed, or references missing/unreadable + * certificate or private key files. This is treated as an unrecoverable misconfiguration. + *
  3. {@code null} (Safe fallback / fail open): Client certificates are explicitly + * disabled via {@code GOOGLE_API_USE_CLIENT_CERTIFICATE} (any non-empty value other than + * {@code "true"}), no explicit configuration is set and neither the default well-known + * configuration file nor a readable GKE credential bundle exists on disk, or the + * configuration specifies an non-workload source (e.g., ECP/PKCS11 without a {@code + * workload} section). Callers can proceed without workload certificate file polling. + *
+ */ + public static @Nullable String getWorkloadCertPath( + EnvironmentProvider envProvider, PropertyProvider propProvider) { + String rawVal = envProvider.getEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE"); + String useClientCertificate = rawVal != null ? rawVal.trim() : null; + if (isUseClientCertificateExplicitlyDisabled(useClientCertificate)) { + return null; + } + + String explicitConfigPath = envProvider.getEnv(CERTIFICATE_CONFIGURATION_ENV_VARIABLE); + + // 1. Explicit Configuration Path (Fail Closed) + if (!Strings.isNullOrEmpty(explicitConfigPath)) { + File configFile = new File(explicitConfigPath); + if (!configFile.exists()) { + throw new IllegalStateException( + "Certificate configuration file specified via GOOGLE_API_CERTIFICATE_CONFIG at '" + + explicitConfigPath + + "' does not exist."); + } + if (!configFile.isFile() || !configFile.canRead()) { + throw new IllegalStateException( + "Failed to read certificate configuration file specified via" + + " GOOGLE_API_CERTIFICATE_CONFIG at '" + + explicitConfigPath + + "'."); + } + WorkloadCertificateConfiguration config; + try { + config = getWorkloadCertificateConfiguration(envProvider, propProvider, explicitConfigPath); + } catch (CertificateSourceUnavailableException e) { + // ECP / PKCS11 configuration without workload section; safe fallback + return null; + } catch (Exception e) { + throw new IllegalStateException( + "Certificate configuration file specified via GOOGLE_API_CERTIFICATE_CONFIG at '" + + explicitConfigPath + + "' is malformed: " + + e.getMessage(), + e); + } + checkCertAndKeyFilesReadable(config, explicitConfigPath, false); + return config.getCertPath(); + } + + // 2. Implicit / Default gcloud Configuration Path + File defaultConfigFile = null; + try { + defaultConfigFile = getWellKnownCertificateConfigFile(envProvider, propProvider); + } catch (IOException e) { + // APPDATA missing on Windows, etc. Safe fallback. + } + if (defaultConfigFile != null && defaultConfigFile.exists()) { + if (!defaultConfigFile.isFile() || !defaultConfigFile.canRead()) { + throw new IllegalStateException( + "Default certificate configuration file at '" + + defaultConfigFile.getAbsolutePath() + + "' exists but could not be read."); + } + WorkloadCertificateConfiguration config = null; + try { + config = getWorkloadCertificateConfiguration(envProvider, propProvider, null); + } catch (CertificateSourceUnavailableException e) { + // ECP-only configuration without workload section; safe fallback + } catch (Exception e) { + throw new IllegalStateException( + "Default certificate configuration file at '" + + defaultConfigFile.getAbsolutePath() + + "' is malformed: " + + e.getMessage(), + e); + } + if (config != null) { + checkCertAndKeyFilesReadable(config, defaultConfigFile.getAbsolutePath(), true); + return config.getCertPath(); + } + // A default configuration file exists without a workload section (e.g. ECP); it takes + // precedence over the GKE credential bundle. + return null; + } + + // 3. GKE credential bundle (Pod Certificates), used only when no certificate configuration + // file is configured or present. The combined bundle serves as both certificate and key. + return findGkeCredentialBundle(); + } + + private static void checkCertAndKeyFilesReadable( + WorkloadCertificateConfiguration config, String configPath, boolean isDefaultConfig) { + File certFile = new File(config.getCertPath()); + File keyFile = new File(config.getPrivateKeyPath()); + if (!certFile.isFile() || !certFile.canRead() || !keyFile.isFile() || !keyFile.canRead()) { + String sourcePrefix = + isDefaultConfig + ? "referenced by default configuration '" + : "referenced by configuration '"; + throw new IllegalStateException( + "Failed to read certificate/key file at '" + + config.getCertPath() + + "' or '" + + config.getPrivateKeyPath() + + "' " + + sourcePrefix + + configPath + + "'."); + } + } + + /** + * Computes the lower-case SHA-256 hex fingerprint of the certificate file at {@code certPath}. + * + *

Unlike {@link #getWorkloadCertPath}, which validates configuration at channel initialization + * and fails closed on errors, this method is called dynamically at runtime during active RPCs to + * detect certificate rotations on disk. External certificate rotators may temporarily delete, + * truncate, or rewrite the certificate file mid-RPC. Returning {@code null} on read/digest + * exceptions (which callers normalize to {@code ""}) allows runtime refresh checks to ignore + * transient mid-write states and keep the active healthy channel without failing in-flight RPCs. + */ + public static @Nullable String getCertificateFingerprint(@Nullable String certPath) { + if (certPath == null) { + return null; + } + try { + byte[] certBytes = Files.readAllBytes(Paths.get(certPath)); + byte[] digest = MessageDigest.getInstance("SHA-256").digest(certBytes); + return BaseEncoding.base16().lowerCase().encode(digest); + } catch (Exception e) { + return null; + } + } + /** * Returns the path to the client certificate file specified by the loaded workload certificate * configuration. @@ -65,14 +318,17 @@ private MtlsUtils() { * @throws IOException if the certificate configuration cannot be found or loaded. */ public static String getCertificatePath( - EnvironmentProvider envProvider, PropertyProvider propProvider, String certConfigPathOverride) + EnvironmentProvider envProvider, + PropertyProvider propProvider, + @Nullable String certConfigPathOverride) throws IOException { String certPath = getWorkloadCertificateConfiguration(envProvider, propProvider, certConfigPathOverride) .getCertPath(); if (Strings.isNullOrEmpty(certPath)) { throw new CertificateSourceUnavailableException( - "Certificate configuration loaded successfully, but does not contain a 'certificate_file' path."); + "Certificate configuration loaded successfully, but does not contain a" + + " 'cert_configs.workload.cert_path' path."); } return certPath; } @@ -92,7 +348,9 @@ public static String getCertificatePath( * @throws IOException if the configuration file cannot be found, read, or parsed */ static WorkloadCertificateConfiguration getWorkloadCertificateConfiguration( - EnvironmentProvider envProvider, PropertyProvider propProvider, String certConfigPathOverride) + EnvironmentProvider envProvider, + PropertyProvider propProvider, + @Nullable String certConfigPathOverride) throws IOException { File certConfig; if (certConfigPathOverride != null) { @@ -106,7 +364,7 @@ static WorkloadCertificateConfiguration getWorkloadCertificateConfiguration( } } - if (!certConfig.isFile()) { + if (!certConfig.isFile() || !certConfig.canRead()) { throw new CertificateSourceUnavailableException( "Certificate configuration file does not exist or is not a file: " + certConfig.getAbsolutePath()); diff --git a/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/SecureConnectProvider.java b/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/SecureConnectProvider.java index 06294c611c7b..b030a992a9dd 100644 --- a/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/SecureConnectProvider.java +++ b/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/SecureConnectProvider.java @@ -32,10 +32,11 @@ import com.google.api.client.json.JsonParser; import com.google.api.client.json.gson.GsonFactory; -import com.google.api.client.util.SecurityUtils; import com.google.api.core.InternalApi; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; @@ -147,8 +148,25 @@ static KeyStore getKeyStore(InputStream metadata, ProcessProvider processProvide "SecureConnect: Cert provider command failed with exit code: " + exitCode); } - // Create mTLS key store with the input certificates from shell command. - return SecurityUtils.createMtlsKeyStore(process.getInputStream()); + // Create mTLS key store with the input certificates from shell command. Read only the bytes + // already buffered now that the process has exited, so a background child process holding + // stdout open cannot block waiting for EOF. + ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + try (InputStream processOut = process.getInputStream()) { + if (processOut != null) { + for (int available = processOut.available(); + available > 0; + available = processOut.available()) { + byte[] buffer = new byte[available]; + int read = processOut.read(buffer); + if (read <= 0) { + break; + } + stdout.write(buffer, 0, read); + } + } + } + return MtlsKeyStoreUtils.createMtlsKeyStore(new ByteArrayInputStream(stdout.toByteArray())); } @VisibleForTesting diff --git a/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/X509Provider.java b/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/X509Provider.java index 911c0fdcf922..fa7deeda21ca 100644 --- a/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/X509Provider.java +++ b/google-auth-library-java/oauth2_http/java/com/google/auth/mtls/X509Provider.java @@ -30,7 +30,6 @@ package com.google.auth.mtls; -import com.google.api.client.util.SecurityUtils; import com.google.api.core.InternalApi; import com.google.auth.oauth2.EnvironmentProvider; import com.google.auth.oauth2.PropertyProvider; @@ -38,6 +37,7 @@ import com.google.auth.oauth2.SystemPropertyProvider; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.SequenceInputStream; @@ -54,7 +54,8 @@ public class X509Provider implements MtlsProvider { private final EnvironmentProvider envProvider; private final PropertyProvider propProvider; - private final String certConfigPathOverride; + private final @Nullable String certConfigPathOverride; + private final boolean allowGkeCredentialBundle; /** * Creates an X509 provider with an override path for the certificate configuration, bypassing the @@ -70,9 +71,39 @@ public X509Provider( EnvironmentProvider envProvider, PropertyProvider propProvider, String certConfigPathOverride) { + this(envProvider, propProvider, certConfigPathOverride, false); + } + + /** + * Creates an X509 provider that optionally falls back to the GKE credential bundle when no + * certificate configuration is present. + * + * @param envProvider environment provider used for environment variables + * @param propProvider property provider used for system properties + * @param certConfigPathOverride the path to read the certificate configuration from. + * @param allowGkeCredentialBundle whether to fall back to the GKE credential bundle when no + * certificate configuration is present and {@code certConfigPathOverride} is null. + */ + X509Provider( + EnvironmentProvider envProvider, + PropertyProvider propProvider, + @Nullable String certConfigPathOverride, + boolean allowGkeCredentialBundle) { this.envProvider = envProvider; this.propProvider = propProvider; this.certConfigPathOverride = certConfigPathOverride; + this.allowGkeCredentialBundle = allowGkeCredentialBundle; + } + + /** + * Creates an X.509 provider that, in addition to the locations checked by {@link + * #X509Provider()}, falls back to the GKE credential bundle when no certificate configuration is + * present, so that the transport uses the same certificate that {@link + * MtlsUtils#getWorkloadCertPath} reports for certificate-bound tokens. + */ + static X509Provider createWithGkeCredentialBundleFallback() { + return new X509Provider( + SystemEnvironmentProvider.getInstance(), SystemPropertyProvider.getInstance(), null, true); } /** @@ -105,18 +136,62 @@ public X509Provider() { *

  • The certificate config override path, if set. *
  • The path pointed to by the "GOOGLE_API_CERTIFICATE_CONFIG" environment variable *
  • The well known gcloud location for the certificate configuration file. + *
  • If none of the above is configured or present, and this provider was created with {@link + * #createWithGkeCredentialBundleFallback()}, the GKE credential bundle at {@code + * /var/run/secrets/workload-spiffe-credentials/x509.credential-bundle.private-key.pem}, + * which contains both the certificate chain and the private key. * * - * @return a KeyStore containing the X.509 certificate specified by the certificate configuration. + * @return a KeyStore containing the X.509 certificate chain and private key from the certificate + * source above. * @throws CertificateSourceUnavailableException if the certificate source is unavailable (ex. * missing configuration file) * @throws IOException if a general I/O error occurs while creating the KeyStore */ @Override public KeyStore getKeyStore() throws CertificateSourceUnavailableException, IOException { - WorkloadCertificateConfiguration workloadCertConfig = - MtlsUtils.getWorkloadCertificateConfiguration( - envProvider, propProvider, certConfigPathOverride); + WorkloadCertificateConfiguration workloadCertConfig; + try { + workloadCertConfig = + MtlsUtils.getWorkloadCertificateConfiguration( + envProvider, propProvider, certConfigPathOverride); + } catch (CertificateSourceUnavailableException configUnavailable) { + // Only fall back to the GKE credential bundle when created for the mTLS transport + // (allowGkeCredentialBundle == true, rather than public X.509 WIF constructors) and when no + // explicit certificate configuration path override was passed to the constructor. + if (allowGkeCredentialBundle && certConfigPathOverride == null) { + // getWorkloadCertificateConfiguration also throws CertificateSourceUnavailableException + // when GOOGLE_API_CERTIFICATE_CONFIG is explicitly set or when a well-known + // certificate_config.json exists with only a non-workload (e.g. ECP) configuration; + // getGkeCredentialBundlePath checks hasCertificateConfiguration so we do not fall back to + // the GKE bundle in those cases. + String gkeCredentialBundlePath = + MtlsUtils.getGkeCredentialBundlePath(envProvider, propProvider); + if (gkeCredentialBundlePath != null) { + try (InputStream bundleStream = new FileInputStream(new File(gkeCredentialBundlePath))) { + return MtlsKeyStoreUtils.createMtlsKeyStore(bundleStream); + } catch (FileNotFoundException e) { + // The bundle was removed (e.g. mid-rotation) after the availability check. + throw new CertificateSourceUnavailableException( + "GKE credential bundle at " + + gkeCredentialBundlePath + + " was removed before it could be read (for example, during certificate" + + " rotation). Retry the request or verify that the GKE Pod Certificate volume" + + " is mounted.", + e); + } catch (Exception e) { + throw new IOException( + "X509Provider: Unable to load GKE credential bundle at " + + gkeCredentialBundlePath + + ". Verify that the file contains a valid PEM-encoded X.509 certificate chain" + + " and PKCS#8 private key, or set GOOGLE_API_USE_CLIENT_CERTIFICATE=false to" + + " disable mTLS.", + e); + } + } + } + throw configUnavailable; + } // Read the certificate and private key file paths into streams. try (InputStream certStream = new FileInputStream(new File(workloadCertConfig.getCertPath())); @@ -126,7 +201,7 @@ public KeyStore getKeyStore() throws CertificateSourceUnavailableException, IOEx new SequenceInputStream(certStream, privateKeyStream)) { // Build a key store using the combined stream. - return SecurityUtils.createMtlsKeyStore(certAndPrivateKeyStream); + return MtlsKeyStoreUtils.createMtlsKeyStore(certAndPrivateKeyStream); } catch (CertificateSourceUnavailableException e) { // Throw the CertificateSourceUnavailableException without wrapping. throw e; diff --git a/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/AgentIdentityCacheUtils.java b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/AgentIdentityCacheUtils.java new file mode 100644 index 000000000000..0dfa9d63c8a4 --- /dev/null +++ b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/AgentIdentityCacheUtils.java @@ -0,0 +1,377 @@ +/* + * Copyright 2026, Google Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.auth.oauth2; + +import com.google.auth.oauth2.AgentIdentityUtils.CertInfo; +import com.google.common.base.Strings; +import java.io.IOException; +import java.nio.file.AccessDeniedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.util.Objects; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +/** + * Utility class for in-memory caching and filesystem metadata validation of Agent Identity + * certificates and configuration files. + */ +@NullMarked +final class AgentIdentityCacheUtils { + + // In-memory cache of verified Agent Identity info to avoid redundant disk reads, X.509/PKCS#8 + // parsing, and cryptographic signature verification on every token refresh when files are + // unchanged. + private static volatile @Nullable CachedAgentIdentityInfo cachedAgentIdentityInfo; + + private AgentIdentityCacheUtils() {} + + /** + * Captures filesystem metadata (path, modification time, size, and OS file key/inode) for a + * certificate, key, or configuration file to detect rotation on disk without re-parsing unchanged + * files. + */ + static final class FileMetadata { + private final String path; + private final FileTime lastModifiedTime; + private final long size; + private final @Nullable Object fileKey; + + private FileMetadata( + final String path, + final FileTime lastModifiedTime, + final long size, + final @Nullable Object fileKey) { + this.path = path; + this.lastModifiedTime = lastModifiedTime; + this.size = size; + this.fileKey = fileKey; + } + + /** Returns the filesystem path captured by this metadata snapshot. */ + String getPath() { + return path; + } + + /** Reads the current filesystem attributes for the given file path. */ + static @Nullable FileMetadata of(final @Nullable String pathStr) throws IOException { + if (Strings.isNullOrEmpty(pathStr)) { + return null; + } + Path path = Paths.get(pathStr); + BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); + return new FileMetadata(pathStr, attrs.lastModifiedTime(), attrs.size(), attrs.fileKey()); + } + + /** Returns true if the given metadata matches this file's path, mtime, size, and file key. */ + boolean matches(final @Nullable FileMetadata other) { + if (other == null) { + return false; + } + return Objects.equals(this.path, other.path) + && Objects.equals(this.lastModifiedTime, other.lastModifiedTime) + && this.size == other.size + && Objects.equals(this.fileKey, other.fileKey); + } + + /** + * Re-reads the file's current attributes from disk and returns {@code true} if the file still + * exists on disk with the exact same modification time, size, and OS file key/inode as when + * this snapshot was captured. + */ + boolean isUnchangedOnDisk() { + try { + return matches(FileMetadata.of(this.path)); + } catch (IOException e) { + return false; + } + } + + /** + * Returns {@code true} if {@code expectedPath} equals this snapshot's path and the file on disk + * is unchanged since this snapshot was captured. + */ + boolean isUnchangedAtPath(final @Nullable String expectedPath) { + return Objects.equals(this.path, expectedPath) && isUnchangedOnDisk(); + } + } + + /** + * Stores an in-memory snapshot of verified Agent Identity certificate information and its + * associated filesystem metadata to avoid redundant disk I/O and cryptographic checks on + * subsequent token refreshes. + */ + static final class CachedAgentIdentityInfo { + /** + * Filesystem metadata of the JSON certificate configuration file ({@code + * GOOGLE_API_CERTIFICATE_CONFIG}) when it was parsed, or {@code null} if the certificate was + * discovered from the well-known directory. + */ + final @Nullable FileMetadata configMetadata; + + /** + * Filesystem metadata of the X.509 certificate or combined credential bundle file when it was + * read. + */ + final @Nullable FileMetadata certMetadata; + + /** + * Filesystem metadata of the private key or combined credential bundle file when it was read, + * or {@code null} when {@link #shouldRequestBoundToken} is {@code false} (for non-agent + * certificates where the private key is not read). + */ + final @Nullable FileMetadata keyMetadata; + + /** + * Whether the parsed certificate's Subject Alternative Names matched an allowed Agent Identity + * SPIFFE trust domain pattern and therefore requires requesting a bound token. + */ + final boolean shouldRequestBoundToken; + + /** + * The verified {@link AgentIdentityUtils.CertInfo} (parsed {@code X509Certificate} and PEM + * certificate chain) when {@link #shouldRequestBoundToken} is {@code true}, or {@code null} + * when {@link #shouldRequestBoundToken} is {@code false}. + */ + final @Nullable CertInfo certInfo; + + /** + * Whether {@link #certMetadata} points to the well-known GKE credential bundle path ({@link + * AgentIdentityUtils#isGkeCredentialBundle}). + */ + final boolean isGkeCredentialBundle; + + /** + * Constructs a cached snapshot of Agent Identity evaluation state and file metadata. + * + * @param configMetadata filesystem metadata of the certificate config file, or {@code null} + * when discovered from the well-known directory + * @param certMetadata filesystem metadata of the certificate or bundle file + * @param keyMetadata filesystem metadata of the private key or bundle file, or {@code null} + * when {@code shouldRequestBoundToken} is {@code false} + * @param shouldRequestBoundToken {@code true} if the certificate matched an Agent Identity + * SPIFFE pattern and requires a bound token; {@code false} otherwise + * @param certInfo the verified {@link AgentIdentityUtils.CertInfo} when {@code + * shouldRequestBoundToken} is {@code true}, or {@code null} otherwise + */ + CachedAgentIdentityInfo( + final @Nullable FileMetadata configMetadata, + final @Nullable FileMetadata certMetadata, + final @Nullable FileMetadata keyMetadata, + final boolean shouldRequestBoundToken, + final @Nullable CertInfo certInfo) { + this.configMetadata = configMetadata; + this.certMetadata = certMetadata; + this.keyMetadata = keyMetadata; + this.shouldRequestBoundToken = shouldRequestBoundToken; + this.certInfo = certInfo; + this.isGkeCredentialBundle = + certMetadata != null && AgentIdentityUtils.isGkeCredentialBundle(certMetadata.getPath()); + } + + /** + * Returns {@code true} if this cached entry was discovered from the well-known directory (not a + * certificate config file) and has both certificate and key metadata available for fallback. + */ + boolean canUseWellKnownFallback() { + return configMetadata == null && certMetadata != null && keyMetadata != null; + } + } + + /** Returns the current in-memory cached Agent Identity info, or {@code null} if none. */ + static @Nullable CachedAgentIdentityInfo getCachedAgentIdentityInfo() { + return cachedAgentIdentityInfo; + } + + /** Clears the in-memory cached Agent Identity info. */ + static void clearCachedAgentIdentityInfo() { + cachedAgentIdentityInfo = null; + } + + /** + * Returns {@code true} if the cached snapshot is non-null and its config, certificate, and key + * file metadata still match the current files on disk. + */ + static boolean isCachedInfoValid( + final @Nullable CachedAgentIdentityInfo cached, final @Nullable String certConfigPath) { + if (cached == null || cached.certMetadata == null) { + return false; + } + boolean hasConfigEnv = !Strings.isNullOrEmpty(certConfigPath); + if (hasConfigEnv) { + // When GOOGLE_API_CERTIFICATE_CONFIG is set, the cache must have been populated from that + // config file (configMetadata != null) and the config file on disk must be unchanged. + if (cached.configMetadata == null + || !cached.configMetadata.isUnchangedAtPath(certConfigPath)) { + return false; + } + } else { + // When GOOGLE_API_CERTIFICATE_CONFIG is unset, the cache must have been populated from the + // well-known directory (where configMetadata is null). + if (cached.configMetadata != null) { + return false; + } + if (cached.certMetadata.path.endsWith("certificates.pem") + && Files.exists( + Paths.get(AgentIdentityUtils.getWellKnownDir(), "credentialbundle.pem"))) { + // credentialbundle.pem takes precedence if added after certificates.pem was cached + return false; + } + } + if (!cached.certMetadata.isUnchangedOnDisk()) { + return false; + } + if (!cached.shouldRequestBoundToken) { + return true; + } + return cached.keyMetadata != null && cached.keyMetadata.isUnchangedOnDisk(); + } + + /** + * Checks whether the resolved {@code certPath}, {@code keyPath}, and {@code configMetaBefore} + * match the given cached snapshot. + * + *

    Even after {@link #isCachedInfoValid} returns {@code false} at the start of {@code + * AgentIdentityUtils.getAgentIdentityCertInfo()}, this check can still hit in three cases: + * + *

      + *
    1. The config file was deleted after caching while the cert and key files on disk remain + * unchanged ({@code getPathsFromConfigWithRetry} fell back to the cached paths and {@code + * configMetadata}, where {@code tryUpdateCache} will not overwrite the cache while the + * config file is missing). + *
    2. Another thread completed rotation and updated {@code cachedAgentIdentityInfo} while the + * current thread was resolving paths. + *
    3. Package-private {@code AgentIdentityUtils.loadAndVerifyCredentials(certPath, keyPath)} + * was invoked directly without going through {@code getAgentIdentityCertInfo()}. + *
    + */ + static boolean isPostResolutionCacheHit( + final @Nullable CachedAgentIdentityInfo cached, + final @Nullable String certPath, + final @Nullable String keyPath, + final @Nullable FileMetadata configMetaBefore) { + if (cached == null || cached.certMetadata == null) { + return false; + } + boolean configMatches = + configMetaBefore == null + ? cached.configMetadata == null + : configMetaBefore.matches(cached.configMetadata); + if (!configMatches || !cached.certMetadata.isUnchangedAtPath(certPath)) { + return false; + } + if (!cached.shouldRequestBoundToken) { + return true; + } + return !Strings.isNullOrEmpty(keyPath) + && cached.keyMetadata != null + && cached.keyMetadata.isUnchangedAtPath(keyPath); + } + + /** + * Returns the latest {@link CachedAgentIdentityInfo} if it contains a verified bound-token {@link + * AgentIdentityUtils.CertInfo}, or falls back to {@code initialCached}. + * + *

    {@code initialCached} is captured at the start of {@code getAgentIdentityCertInfo()} so that + * if the cache is cleared or overwritten with a non-agent certificate during retry backoff, the + * caller still retains the last-known-good bound certificate snapshot, while preferring a fresher + * bound certificate if another thread completed rotation in the meantime. + */ + static @Nullable CachedAgentIdentityInfo getLatestOrInitialCache( + final @Nullable CachedAgentIdentityInfo initialCached) { + CachedAgentIdentityInfo latest = cachedAgentIdentityInfo; + return (latest != null && latest.shouldRequestBoundToken && latest.certInfo != null) + ? latest + : initialCached; + } + + /** + * Safely updates the in-memory cache only if file metadata remained stable across the read/verify + * operation. Returns {@code true} if the post-read cert/key metadata matched the pre-read + * metadata, or {@code false} if the cert or key changed or disappeared mid-read. If only the + * config file changed, returns {@code true} without updating the cache, so the next refresh + * re-resolves paths from the new config. + */ + static boolean tryUpdateCache( + final @Nullable FileMetadata configMetaBefore, + final @Nullable FileMetadata certMetaBefore, + final @Nullable FileMetadata keyMetaBefore, + final boolean shouldRequestBoundToken, + final @Nullable CertInfo certInfo) { + if (certMetaBefore == null || !certMetaBefore.isUnchangedOnDisk()) { + return false; + } + boolean configStable = configMetaBefore == null || configMetaBefore.isUnchangedOnDisk(); + if (!shouldRequestBoundToken) { + if (configStable) { + cachedAgentIdentityInfo = + new CachedAgentIdentityInfo(configMetaBefore, certMetaBefore, null, false, null); + } + return true; + } + if (keyMetaBefore != null && keyMetaBefore.isUnchangedOnDisk()) { + if (configStable) { + cachedAgentIdentityInfo = + new CachedAgentIdentityInfo( + configMetaBefore, certMetaBefore, keyMetaBefore, true, certInfo); + } + return true; + } + return false; + } + + /** + * Checks if a file exists, is a non-empty regular file, and is readable, throwing {@link + * AccessDeniedException} if permission is denied. + */ + static boolean checkExistsOrAccessDenied(final Path path) throws AccessDeniedException { + try { + BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); + if (!attrs.isRegularFile()) { + return false; + } + if (!Files.isReadable(path)) { + if (!Files.exists(path)) { + return false; // Deleted mid-rotation; let the caller retry. + } + throw new AccessDeniedException(path.toString()); + } + return attrs.size() > 0; + } catch (AccessDeniedException e) { + throw e; + } catch (IOException e) { + return false; + } + } +} diff --git a/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/AgentIdentityCertificateValidationUtils.java b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/AgentIdentityCertificateValidationUtils.java new file mode 100644 index 000000000000..e39b9bc36480 --- /dev/null +++ b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/AgentIdentityCertificateValidationUtils.java @@ -0,0 +1,248 @@ +/* + * Copyright 2026, Google Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.auth.oauth2; + +import com.google.common.collect.ImmutableList; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.Signature; +import java.security.cert.CertificateFactory; +import java.security.cert.CertificateParsingException; +import java.security.cert.X509Certificate; +import java.security.spec.X509EncodedKeySpec; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.logging.Level; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +/** + * Utility class for parsing, validating, and verifying X.509 certificates, SPIFFE IDs, and PKCS#8 + * private keys for Agent Identity token binding. + */ +@NullMarked +final class AgentIdentityCertificateValidationUtils { + + private static final LoggerProvider LOGGER_PROVIDER = + LoggerProvider.forClazz(AgentIdentityCertificateValidationUtils.class); + + // Allowed SPIFFE trust domain patterns for agentic identities. + private static final List AGENT_IDENTITY_SPIFFE_PATTERNS = + ImmutableList.of( + Pattern.compile("^agents\\.global\\.org-\\d+\\.system\\.id\\.goog$"), + Pattern.compile("^agents\\.global\\.proj-\\d+\\.system\\.id\\.goog$"), + Pattern.compile("^agents-nonprod\\.global\\.org-\\d+\\.system\\.id\\.goog$"), + Pattern.compile("^agents-nonprod\\.global\\.proj-\\d+\\.system\\.id\\.goog$")); + + // Subject Alternative Name (SAN) type for URI as defined in RFC 5280 Section 4.2.1.6. + private static final int SAN_URI_TYPE = 6; + + private static final String SPIFFE_SCHEME_PREFIX = "spiffe://"; + + private static final byte[] VERIFICATION_DATA = + "verification-data".getBytes(StandardCharsets.UTF_8); + + private AgentIdentityCertificateValidationUtils() {} + + /** Thrown for key algorithms that can never be verified, so callers fail without retrying. */ + static final class UnsupportedKeyAlgorithmException extends IOException { + private static final long serialVersionUID = 1L; + + UnsupportedKeyAlgorithmException(final @Nullable String algorithm) { + super("Unsupported key algorithm: " + algorithm); + } + } + + /** + * Thrown when a certificate PEM block contains malformed X.509 data, so callers fail fast without + * retrying. + */ + static final class InvalidCertificateException extends IOException { + private static final long serialVersionUID = 1L; + + InvalidCertificateException(final String message, final Throwable cause) { + super(message, cause); + } + } + + /** + * Reads the full certificate chain from the specified path as a PEM string. + * + *

    Extracts only the {@code -----BEGIN CERTIFICATE-----} blocks using {@link + * OAuth2Utils#PEM_CERT_PATTERN}, stripping any private keys or non-certificate data that may be + * present in a combined bundle file. + */ + static String readCertificateChain(final String certPath) throws IOException { + byte[] certData = Files.readAllBytes(Paths.get(certPath)); + String content = new String(certData, StandardCharsets.UTF_8); + Matcher matcher = OAuth2Utils.PEM_CERT_PATTERN.matcher(content); + StringBuilder certChain = new StringBuilder(); + while (matcher.find()) { + certChain.append(matcher.group(0)).append("\n"); + } + if (certChain.length() == 0) { + throw new IOException("No PEM certificates found in certificate file: " + certPath); + } + return certChain.toString(); + } + + /** Parses the X509 certificate from the specified content string. */ + static X509Certificate parseCertificateContent(final String certContent) throws IOException { + try (InputStream stream = + new ByteArrayInputStream(certContent.getBytes(StandardCharsets.UTF_8))) { + CertificateFactory cf = CertificateFactory.getInstance("X.509"); + return (X509Certificate) cf.generateCertificate(stream); + } catch (GeneralSecurityException e) { + throw new InvalidCertificateException( + "Failed to parse Agent Identity certificate for bound token request.", e); + } + } + + /** Reads the private key from the specified path using PKCS8 format. */ + static PrivateKey readPrivateKey(final String keyPath, final @Nullable String algorithm) + throws IOException { + String keyPem = new String(Files.readAllBytes(Paths.get(keyPath)), StandardCharsets.UTF_8); + OAuth2Utils.Pkcs8Algorithm pkcs8Alg; + if ("EC".equals(algorithm) || "ECDSA".equals(algorithm)) { + pkcs8Alg = OAuth2Utils.Pkcs8Algorithm.EC; + } else if ("RSA".equals(algorithm)) { + pkcs8Alg = OAuth2Utils.Pkcs8Algorithm.RSA; + } else { + throw new UnsupportedKeyAlgorithmException(algorithm); + } + return OAuth2Utils.privateKeyFromPkcs8(keyPem, pkcs8Alg); + } + + /** + * Verifies that the private key corresponds to the public key in the certificate by performing a + * test signature and verification. + */ + static boolean verifyKeyPair(final X509Certificate cert, final PrivateKey privateKey) + throws UnsupportedKeyAlgorithmException { + PublicKey publicKey = cert.getPublicKey(); + String keyAlgorithm = publicKey.getAlgorithm(); + String sigAlg; + if ("RSA".equals(keyAlgorithm)) { + sigAlg = "SHA256withRSA"; + } else if ("EC".equals(keyAlgorithm) || "ECDSA".equals(keyAlgorithm)) { + sigAlg = "SHA256withECDSA"; + } else { + throw new UnsupportedKeyAlgorithmException(keyAlgorithm); + } + + try { + if ("ECDSA".equals(keyAlgorithm) && publicKey.getEncoded() != null) { + // SunEC rejects keys whose algorithm is "ECDSA" rather than "EC"; normalize the key. + publicKey = + KeyFactory.getInstance("EC") + .generatePublic(new X509EncodedKeySpec(publicKey.getEncoded())); + } + + Signature signer = Signature.getInstance(sigAlg); + signer.initSign(privateKey); + signer.update(VERIFICATION_DATA); + byte[] signature = signer.sign(); + + Signature verifier = Signature.getInstance(sigAlg); + verifier.initVerify(publicKey); + verifier.update(VERIFICATION_DATA); + + return verifier.verify(signature); + } catch (Exception e) { + LoggingUtils.log( + LOGGER_PROVIDER, + Level.FINE, + Collections.emptyMap(), + "Key pair verification failed: " + e.getMessage()); + return false; + } + } + + /** + * Determines if a bound token should be requested by checking if any of the certificate's Subject + * Alternative Names (SANs) match allowed SPIFFE patterns. + */ + static boolean shouldRequestBoundToken(final X509Certificate cert) { + Collection> sans; + try { + sans = cert.getSubjectAlternativeNames(); + } catch (CertificateParsingException e) { + LoggingUtils.log( + LOGGER_PROVIDER, + Level.FINE, + Collections.emptyMap(), + "Failed to parse Subject Alternative Names from certificate: " + e.getMessage()); + return false; + } + if (sans == null) { + return false; + } + // Iterate through all Subject Alternative Names + for (List san : sans) { + // Check if the SAN entry is a URI (type 6) with a String value + if (san.size() < 2 + || !(san.get(0) instanceof Integer) + || (Integer) san.get(0) != SAN_URI_TYPE + || !(san.get(1) instanceof String)) { + continue; + } + String uri = ((String) san.get(1)).toLowerCase(Locale.US); + // Check if the URI starts with "spiffe://" + if (!uri.startsWith(SPIFFE_SCHEME_PREFIX)) { + continue; + } + String withoutScheme = uri.substring(SPIFFE_SCHEME_PREFIX.length()); + int slashIndex = withoutScheme.indexOf('/'); + // Extract the trust domain (part before the first slash) + String trustDomain = + (slashIndex == -1) ? withoutScheme : withoutScheme.substring(0, slashIndex); + // Match the trust domain against allowed agent patterns + for (Pattern pattern : AGENT_IDENTITY_SPIFFE_PATTERNS) { + if (pattern.matcher(trustDomain).matches()) { + return true; + } + } + } + return false; + } +} diff --git a/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/AgentIdentityUtils.java b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/AgentIdentityUtils.java new file mode 100644 index 000000000000..2bbe599df301 --- /dev/null +++ b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/AgentIdentityUtils.java @@ -0,0 +1,964 @@ +/* + * Copyright 2026, Google Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.auth.oauth2; + +import com.google.api.client.json.GenericJson; +import com.google.api.client.json.JsonObjectParser; +import com.google.auth.mtls.MtlsUtils; +import com.google.auth.oauth2.AgentIdentityCacheUtils.CachedAgentIdentityInfo; +import com.google.auth.oauth2.AgentIdentityCacheUtils.FileMetadata; +import com.google.auth.oauth2.AgentIdentityCertificateValidationUtils.InvalidCertificateException; +import com.google.auth.oauth2.AgentIdentityCertificateValidationUtils.UnsupportedKeyAlgorithmException; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.AccessDeniedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.Collections; +import java.util.Map; +import java.util.logging.Level; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +/** Utility class for Agent Identity runtime certificate discovery and token binding. */ +@NullMarked +final class AgentIdentityUtils { + + private static final LoggerProvider LOGGER_PROVIDER = + LoggerProvider.forClazz(AgentIdentityUtils.class); + + /** + * Environment variable pointing to the client certificate configuration file. + * + *

    If set, certificate and key paths are resolved from the configuration file specified by this + * variable. + */ + static final String GOOGLE_API_CERTIFICATE_CONFIG = "GOOGLE_API_CERTIFICATE_CONFIG"; + + /** + * Environment variable to explicitly enable or disable runtime token binding. Defaults to true if + * unset. + */ + static final String GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN = + "GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN"; + + /** + * Legacy Cloud Run environment variable to prevent agent token sharing for GCP services. Used as + * a fallback if {@link #GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN} is unset. + */ + static final String GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES = + "GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES"; + + /** + * Environment variable to explicitly enable or disable client certificate authentication (mTLS). + * + *

    When set to {@code "true"}, mTLS is enforced. When set to any other non-empty value (such as + * {@code "false"}), mTLS and token binding are disabled. + */ + static final String GOOGLE_API_USE_CLIENT_CERTIFICATE = "GOOGLE_API_USE_CLIENT_CERTIFICATE"; + + private static volatile String wellKnownDir = "/var/run/secrets/workload-spiffe-credentials/"; + + /** + * File name of the combined certificate chain and private key bundle that GKE delivers to the + * well-known directory for pods with Agent Identity (Pod Certificates). + * + *

    When {@link #GOOGLE_API_CERTIFICATE_CONFIG} is unset and no well-known gcloud certificate + * configuration file exists, the presence of this bundle auto-enables mTLS on its own (no {@link + * #GOOGLE_API_USE_CLIENT_CERTIFICATE} required). It is never polled for, because GKE delivers it + * before the container starts and polling on its absence would delay every workload without Agent + * Identity. It must be a readable, non-empty file. When present and allowed, it takes precedence + * over the other well-known files, matching the certificate the transport presents. + */ + static final String GKE_CREDENTIAL_BUNDLE_FILE = "x509.credential-bundle.private-key.pem"; + + private static volatile String gkeCredentialBundlePath = + Paths.get(wellKnownDir, GKE_CREDENTIAL_BUNDLE_FILE).toString(); + + // Retries for verifying certificate and private key matching during atomic key rotation. + private static final int CERT_KEY_MATCH_RETRIES = 3; + + private static final long CERT_KEY_MATCH_RETRY_INTERVAL_MS = 100; + + // Polling configuration for initial container startup credential file readiness. + // Matches Python google-auth implementation (_agent_identity_utils.py) for initial + // asynchronous credential delivery (50 * 100ms + 50 * 500ms = 30 seconds total). + private static final int FAST_POLL_CYCLES = 50; + + private static final long FAST_POLL_INTERVAL_MS = 100; // 0.1 seconds + + private static final long SLOW_POLL_INTERVAL_MS = 500; // 0.5 seconds + + private static final long TOTAL_TIMEOUT_MS = 30000; // 30 seconds + + private static final int TOTAL_POLL_CYCLES = + FAST_POLL_CYCLES + + (int) + ((TOTAL_TIMEOUT_MS - (FAST_POLL_CYCLES * FAST_POLL_INTERVAL_MS)) + / SLOW_POLL_INTERVAL_MS); + + private static volatile EnvironmentProvider environmentProvider = + SystemEnvironmentProvider.getInstance(); + + private static volatile PropertyProvider propertyProvider = SystemPropertyProvider.getInstance(); + + private static volatile TimeService timeService = Thread::sleep; + + // Tracks whether initial container startup discovery has completed. The 30-second + // polling loop (TOTAL_POLL_CYCLES) is strictly limited to initial startup so subsequent + // token refreshes never block for 30 seconds. + private static volatile boolean initialStartupCompleted = false; + + // Tracks whether the FINE-level message for explicitly disabled mTLS when certificates are + // present has already been logged in this process to prevent log spam on repeated token + // refreshes. + private static volatile boolean mtlsDisabledLogged = false; + + // Tracks whether mTLS has already been evaluated as explicitly disabled via + // GOOGLE_API_USE_CLIENT_CERTIFICATE, so later refreshes skip re-reading the config from disk. + private static volatile boolean mtlsDisabledEvaluated = false; + + private AgentIdentityUtils() {} + + @VisibleForTesting + @FunctionalInterface + interface TimeService { + /** + * Causes the currently executing thread to sleep for the specified number of milliseconds. + * + * @param millis the length of time to sleep in milliseconds + * @throws InterruptedException if interrupted while sleeping + */ + void sleep(final long millis) throws InterruptedException; + } + + /** + * Holds the parsed {@link X509Certificate} and cached PEM certificate chain content for token + * binding. + * + *

    This class intentionally stores in-memory certificate content rather than file paths to + * prevent redundant filesystem reads and race conditions when binding tokens to requests. + */ + static class CertInfo { + private final X509Certificate certificate; + private final String certContent; + + CertInfo(final X509Certificate certificate, final String certContent) { + this.certificate = certificate; + this.certContent = certContent; + } + + /** Returns the parsed {@link X509Certificate}. */ + X509Certificate getCertificate() { + return certificate; + } + + /** Returns the raw PEM certificate chain content. */ + String getCertContent() { + return certContent; + } + } + + /** Holds the resolved filesystem paths for the certificate and private key. */ + private static final class ResolvedCertAndKeyPaths { + private final @Nullable String certPath; + private final @Nullable String keyPath; + private final boolean hasWorkloadConfig; + private final @Nullable FileMetadata configMetadata; + + ResolvedCertAndKeyPaths( + final @Nullable String certPath, + final @Nullable String keyPath, + final boolean hasWorkloadConfig) { + this(certPath, keyPath, hasWorkloadConfig, null); + } + + ResolvedCertAndKeyPaths( + final @Nullable String certPath, + final @Nullable String keyPath, + final boolean hasWorkloadConfig, + final @Nullable FileMetadata configMetadata) { + this.certPath = certPath; + this.keyPath = keyPath; + this.hasWorkloadConfig = hasWorkloadConfig; + this.configMetadata = configMetadata; + } + + /** Returns the path to the certificate or bundle file. */ + @Nullable String getCertPath() { + return certPath; + } + + /** Returns the path to the private key file, or bundle path if combined. */ + @Nullable String getKeyPath() { + return keyPath; + } + + /** Returns whether a workload configuration was parsed from the certificate config file. */ + boolean hasWorkloadConfig() { + return hasWorkloadConfig; + } + + /** Returns the filesystem metadata of the config file when it was parsed, or null. */ + @Nullable FileMetadata getConfigMetadata() { + return configMetadata; + } + } + + /** + * Thrown when a workload configuration block is present in the certificate config file but is + * missing {@code cert_path} or {@code key_path}, so callers fail fast without polling. + */ + private static final class IncompleteWorkloadConfigException extends IOException { + private static final long serialVersionUID = 1L; + + IncompleteWorkloadConfigException(final String message) { + super(message); + } + } + + private static long getSleepIntervalMs(final int cycle) { + return (cycle < FAST_POLL_CYCLES) ? FAST_POLL_INTERVAL_MS : SLOW_POLL_INTERVAL_MS; + } + + private static @Nullable String getTrimmedEnv(final String name) { + String val = environmentProvider.getEnv(name); + return val != null ? val.trim() : null; + } + + /** + * Reads the trimmed {@link #GOOGLE_API_USE_CLIENT_CERTIFICATE} value from {@link + * #environmentProvider}. Queried via {@code environmentProvider} rather than cached in a {@code + * static final} field so test overrides via {@link #setEnvironmentProvider} take effect. + */ + private static @Nullable String getUseClientCertificateEnv() { + return getTrimmedEnv(GOOGLE_API_USE_CLIENT_CERTIFICATE); + } + + /** + * Returns {@code true} if {@link #GOOGLE_API_USE_CLIENT_CERTIFICATE} is set to {@code "true"} + * (case-insensitive). + */ + private static boolean isMtlsExplicitlyEnabled() { + return "true".equalsIgnoreCase(getUseClientCertificateEnv()); + } + + /** + * Returns {@code true} if {@link #GOOGLE_API_USE_CLIENT_CERTIFICATE} is set to a non-empty value + * other than {@code "true"} (case-insensitive). + */ + private static boolean isMtlsExplicitlyDisabled() { + String useClientCert = getUseClientCertificateEnv(); + return !Strings.isNullOrEmpty(useClientCert) && !"true".equalsIgnoreCase(useClientCert); + } + + /** + * Resolves symlinks on existing ancestor directories of {@code path} (for example, {@code + * /var/run} to {@code /run} before startup files are created), preserving any trailing + * non-existent path elements. + */ + private static Path resolveRealPathAllowingMissing(final Path path) { + Path normalized = path.toAbsolutePath().normalize(); + Path current = normalized; + while (current != null) { + try { + Path realCurrent = current.toRealPath(); + Path relative = current.relativize(normalized); + return relative.toString().isEmpty() + ? realCurrent + : realCurrent.resolve(relative).normalize(); + } catch (IOException e) { + current = current.getParent(); + } + } + return normalized; + } + + /** Checks whether the given path resides within the well-known certificate directory. */ + private static boolean isPathInWellKnownDir(final @Nullable String pathStr) { + if (Strings.isNullOrEmpty(pathStr)) { + return false; + } + try { + Path path = resolveRealPathAllowingMissing(Paths.get(pathStr)); + Path wellKnown = resolveRealPathAllowingMissing(Paths.get(wellKnownDir)); + return path.startsWith(wellKnown); + } catch (Exception e) { + return false; + } + } + + /** Returns {@code true} if {@code path} is the GKE credential bundle in the well-known dir. */ + static boolean isGkeCredentialBundle(final @Nullable String path) { + return path != null && gkeCredentialBundlePath.equals(path); + } + + /** + * Returns whether the resolved certificate paths came from a workload certificate configuration + * or point at the GKE credential bundle (either of which can enable mTLS without {@link + * #GOOGLE_API_USE_CLIENT_CERTIFICATE} being set to {@code "true"}). + */ + private static boolean hasWorkloadConfigOrGkeBundle( + final boolean hasWorkloadConfig, final @Nullable String certPath) { + return hasWorkloadConfig || isGkeCredentialBundle(certPath); + } + + /** + * Retrieves the certificate and raw PEM content for the Agent Identity. + * + *

    This method attempts to load the certificate and private key for the agent identity. It + * first checks the location specified by the {@code GOOGLE_API_CERTIFICATE_CONFIG} environment + * variable. If not set, it falls back to well-known default locations. + * + *

    To handle transient race conditions during certificate rotation on disk, this method employs + * a retry mechanism with backoff when reading the configuration and certificate files. Note that + * callers such as {@link ComputeEngineCredentials#refreshAccessToken()} invoke this method via + * {@link #getBoundTokenPayload()} under {@link OAuth2Credentials}'s token refresh coalescing, + * which already serializes concurrent token refreshes to mitigate thundering herd contention. + * + * @return A {@link CertInfo} object containing the parsed {@link X509Certificate} and its raw PEM + * chain content, or {@code null} if the agent identity features are disabled, opted out, or + * if no valid credentials could be loaded. + * @throws IOException If an I/O error occurs while reading the files, or if the key-pair + * verification fails after retries. + */ + static @Nullable CertInfo getAgentIdentityCertInfo() throws IOException { + if (!isTokenBindingEnabled()) { + return null; + } + if (mtlsDisabledEvaluated && isMtlsExplicitlyDisabled()) { + return null; + } + CachedAgentIdentityInfo initialCached = AgentIdentityCacheUtils.getCachedAgentIdentityInfo(); + String certConfigPath = getTrimmedEnv(GOOGLE_API_CERTIFICATE_CONFIG); + + // Fast-path: check cached info and file metadata before parsing JSON config or paths + if (initialCached != null + && AgentIdentityCacheUtils.isCachedInfoValid(initialCached, certConfigPath)) { + if (!initialCached.shouldRequestBoundToken) { + return null; + } + // isCachedInfoValid() verified that the cached certificate file still exists on disk with + // matching metadata, so certsPresent is guaranteed to be true here. + boolean hasConfigOrGkeBundle = + hasWorkloadConfigOrGkeBundle( + initialCached.configMetadata != null, initialCached.certMetadata.getPath()); + if (!shouldEnableMtls(/* certsPresent= */ true, hasConfigOrGkeBundle)) { + return null; + } + return initialCached.certInfo; + } + + ResolvedCertAndKeyPaths paths = resolveCertAndKeyPaths(certConfigPath, initialCached); + boolean hasConfigOrGkeBundle = + hasWorkloadConfigOrGkeBundle(paths.hasWorkloadConfig(), paths.getCertPath()); + boolean certsPresent = !Strings.isNullOrEmpty(paths.getCertPath()); + + if (!shouldEnableMtls(certsPresent, hasConfigOrGkeBundle)) { + if (isMtlsExplicitlyDisabled()) { + mtlsDisabledEvaluated = true; + } + return null; + } + + return loadAndVerifyCredentials( + paths.getCertPath(), paths.getKeyPath(), paths.getConfigMetadata(), initialCached); + } + + /** + * Resolves the paths for the certificate and private key based on the config path or well-known + * locations. + */ + private static ResolvedCertAndKeyPaths resolveCertAndKeyPaths( + final @Nullable String certConfigPath, final @Nullable CachedAgentIdentityInfo cached) + throws IOException { + if (!Strings.isNullOrEmpty(certConfigPath)) { + // Read cert and key paths from config file. We use retry with backoff to handle + // startup delivery (when in well-known directory) and transient rotation race conditions. + return getPathsFromConfigWithRetry(certConfigPath, cached); + } + // Fallback to well-known locations. We use retry with backoff here as well to handle + // startup delivery (when explicit mTLS is enabled) and race conditions during file + // replacement by a rotation process. + return getWellKnownCertificatePathWithRetry(cached); + } + + /** + * Loads the certificate and private key, and verifies that they form a valid cryptographic + * key-pair, supporting both separate files and combined bundle files. + * + *

    Checks {@link AgentIdentityCertificateValidationUtils#shouldRequestBoundToken} right after + * parsing the certificate content before reading the private key or running signature + * verification. Caches verified results in memory so unchanged files are not re-read and + * re-verified on every token refresh. + * + * @param certPath The path to the certificate or bundle file. + * @param keyPath The path to the private key or bundle file. + * @return A {@link CertInfo} object containing the parsed {@link X509Certificate} and the raw PEM + * certificate chain (with private keys stripped), or {@code null} if the certificate does not + * match agent SPIFFE patterns. + * @throws IOException If the files cannot be read or parsed, or if key-pair verification fails + * after retries. + */ + static @Nullable CertInfo loadAndVerifyCredentials( + final @Nullable String certPath, final @Nullable String keyPath) throws IOException { + return loadAndVerifyCredentials( + certPath, keyPath, null, AgentIdentityCacheUtils.getCachedAgentIdentityInfo()); + } + + private static @Nullable CertInfo loadAndVerifyCredentials( + final @Nullable String certPath, + final @Nullable String keyPath, + final @Nullable FileMetadata configMetaBefore, + final @Nullable CachedAgentIdentityInfo initialCached) + throws IOException { + if (Strings.isNullOrEmpty(certPath)) { + return null; + } + + // Even when isCachedInfoValid() in getAgentIdentityCertInfo() returned false, the cache can + // still match here in three cases: + // 1. The config file was deleted after caching while the cert/key files on disk remain + // unchanged (getPathsFromConfigWithRetry fell back to cached paths + configMetadata). + // 2. Another thread completed rotation and updated the cache while this thread was resolving + // paths in resolveCertAndKeyPaths(). + // 3. Package-private loadAndVerifyCredentials(certPath, keyPath) was invoked directly. + CachedAgentIdentityInfo cached = AgentIdentityCacheUtils.getLatestOrInitialCache(initialCached); + if (AgentIdentityCacheUtils.isPostResolutionCacheHit( + cached, certPath, keyPath, configMetaBefore)) { + return cached.shouldRequestBoundToken ? cached.certInfo : null; + } + + Exception lastException = null; + int retries = 0; + while (retries < CERT_KEY_MATCH_RETRIES) { + try { + FileMetadata certMeta = FileMetadata.of(certPath); + String certContent = AgentIdentityCertificateValidationUtils.readCertificateChain(certPath); + X509Certificate cert = + AgentIdentityCertificateValidationUtils.parseCertificateContent(certContent); + + if (!AgentIdentityCertificateValidationUtils.shouldRequestBoundToken(cert)) { + if (AgentIdentityCacheUtils.tryUpdateCache( + configMetaBefore, certMeta, null, false, null)) { + return null; + } + lastException = new IOException("Certificate file modified during read."); + } else { + FileMetadata keyMeta = FileMetadata.of(keyPath); + PrivateKey privateKey = + AgentIdentityCertificateValidationUtils.readPrivateKey( + keyPath, cert.getPublicKey().getAlgorithm()); + + if (AgentIdentityCertificateValidationUtils.verifyKeyPair(cert, privateKey)) { + CertInfo info = new CertInfo(cert, certContent); + if (AgentIdentityCacheUtils.tryUpdateCache( + configMetaBefore, certMeta, keyMeta, true, info)) { + return info; + } + lastException = new IOException("Certificate or key file modified during read."); + LoggingUtils.log( + LOGGER_PROVIDER, + Level.FINE, + Collections.emptyMap(), + "Cert or key file modified during read, retrying..."); + } else { + lastException = new IOException("Certificate and private key do not match."); + LoggingUtils.log( + LOGGER_PROVIDER, + Level.FINE, + Collections.emptyMap(), + "Cert and key mismatch, retrying..."); + } + } + } catch (AccessDeniedException e) { + throw new IOException( + "Permission denied reading certificate or key files for Agent Identity.", e); + } catch (UnsupportedKeyAlgorithmException | InvalidCertificateException e) { + throw e; + } catch (Exception e) { + lastException = e; + LoggingUtils.log( + LOGGER_PROVIDER, + Level.FINE, + Collections.emptyMap(), + "Failed to read or verify cert/key, retrying: " + e.getMessage()); + } + + retries++; + if (retries < CERT_KEY_MATCH_RETRIES) { + try { + timeService.sleep(CERT_KEY_MATCH_RETRY_INTERVAL_MS); // 0.1 seconds backoff + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while waiting for cert/key match.", e); + } + } + } + + // If files were transiently missing, unreadable, or mid-rotation (lastException != null) + // and we already have a verified credential cached in memory, fall back to cached credentials + // rather than failing or caching an unbound token. Re-read cachedAgentIdentityInfo in case + // another thread completed rotation while this thread was sleeping. + CachedAgentIdentityInfo fallbackCached = + AgentIdentityCacheUtils.getLatestOrInitialCache(initialCached); + if (lastException != null + && fallbackCached != null + && fallbackCached.shouldRequestBoundToken + && fallbackCached.certInfo != null) { + LoggingUtils.log( + LOGGER_PROVIDER, + Level.FINE, + Collections.emptyMap(), + "Agent Identity certificate/key files transiently unavailable during rotation; falling" + + " back to cached credentials."); + return fallbackCached.certInfo; + } + + throw new GoogleAuthException( + true, + CERT_KEY_MATCH_RETRIES, + String.format( + "Agent Identity certificate and private key mismatch or read" + + " failure after %d attempts.", + CERT_KEY_MATCH_RETRIES), + lastException); + } + + /** + * Checks if runtime token binding is enabled. + * + *

    Checks {@link #GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN} first; if unset, falls back to the + * legacy {@link #GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES}. Defaults to {@code + * true} if neither is set. + */ + private static boolean isTokenBindingEnabled() { + String enableRuntimeBoundToken = getTrimmedEnv(GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN); + if (!Strings.isNullOrEmpty(enableRuntimeBoundToken)) { + return !"false".equalsIgnoreCase(enableRuntimeBoundToken); + } + String legacyPreventSharing = + getTrimmedEnv(GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES); + if (!Strings.isNullOrEmpty(legacyPreventSharing)) { + return !"false".equalsIgnoreCase(legacyPreventSharing); + } + return true; + } + + /** + * Reads the certificate path from the config file with retry logic to handle startup delivery and + * rotation race conditions. + */ + private static ResolvedCertAndKeyPaths getPathsFromConfigWithRetry( + final String certConfigPath, final @Nullable CachedAgentIdentityInfo initialCached) + throws IOException { + if (isMtlsExplicitlyDisabled()) { + try { + if (AgentIdentityCacheUtils.checkExistsOrAccessDenied(Paths.get(certConfigPath))) { + ResolvedCertAndKeyPaths paths = extractPathsFromConfig(certConfigPath); + if (paths.hasWorkloadConfig() && Files.exists(Paths.get(paths.getCertPath()))) { + return paths; + } + } + } catch (Exception ignored) { + // Do not fail when mTLS is explicitly disabled + } + return new ResolvedCertAndKeyPaths(null, null, false); + } + + boolean inWellKnownDir = isPathInWellKnownDir(certConfigPath); + boolean shouldPoll = !initialStartupCompleted && inWellKnownDir; + int maxCycles = shouldPoll ? TOTAL_POLL_CYCLES : 1; + boolean warned = false; + IOException lastParseException = null; + + for (int cycle = 0; cycle < maxCycles; cycle++) { + try { + if (AgentIdentityCacheUtils.checkExistsOrAccessDenied(Paths.get(certConfigPath))) { + ResolvedCertAndKeyPaths paths = extractPathsFromConfig(certConfigPath); + if (!paths.hasWorkloadConfig()) { + // Valid non-workload config (e.g. enterprise certs) - exit early without polling! + initialStartupCompleted = true; + return paths; + } + // The config file itself may reside outside wellKnownDir while referencing cert_path or + // key_path inside wellKnownDir that are still being delivered at startup. + if (!initialStartupCompleted + && !shouldPoll + && (isPathInWellKnownDir(paths.getCertPath()) + || isPathInWellKnownDir(paths.getKeyPath()))) { + shouldPoll = true; + maxCycles = TOTAL_POLL_CYCLES; + } + boolean certReady = + AgentIdentityCacheUtils.checkExistsOrAccessDenied(Paths.get(paths.getCertPath())); + boolean keyReady = + AgentIdentityCacheUtils.checkExistsOrAccessDenied(Paths.get(paths.getKeyPath())); + if (certReady && keyReady) { + initialStartupCompleted = true; + return paths; + } + } + } catch (AccessDeniedException e) { + String failedFile = e.getFile() != null ? e.getFile() : certConfigPath; + throw new IOException( + "Permission denied reading certificate config file: " + failedFile, e); + } catch (IncompleteWorkloadConfigException e) { + // Incomplete workload config (missing cert_path or key_path) will not become ready via + // startup polling; break out of the polling loop immediately so initial startup throws + // without waiting 30s (or falls back to a previously cached config in steady state). + lastParseException = e; + break; + } catch (IOException e) { + CachedAgentIdentityInfo latestCached = + AgentIdentityCacheUtils.getLatestOrInitialCache(initialCached); + if (!shouldPoll && cycle + 1 >= maxCycles && latestCached == null) { + initialStartupCompleted = true; + throw e; // Fail fast on malformed JSON syntax errors when not polling + } + lastParseException = e; + // When polling is active, fall through to retry (handles partial/empty JSON during startup) + } + if (cycle + 1 < maxCycles) { + if (shouldPoll && !warned) { + LoggingUtils.log( + LOGGER_PROVIDER, + Level.FINE, + Collections.emptyMap(), + String.format( + "Agent Identity certificate config or referenced credential files not yet ready" + + " at %s (from %s environment variable). Retrying for up to %d seconds.", + certConfigPath, GOOGLE_API_CERTIFICATE_CONFIG, TOTAL_TIMEOUT_MS / 1000)); + warned = true; + } + try { + timeService.sleep(getSleepIntervalMs(cycle)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException( + "Interrupted while waiting for Agent Identity certificate files for bound" + + " token request.", + e); + } + } + } + initialStartupCompleted = true; + + CachedAgentIdentityInfo fallbackCached = + AgentIdentityCacheUtils.getLatestOrInitialCache(initialCached); + if (fallbackCached != null + && fallbackCached.configMetadata != null + && fallbackCached.certMetadata != null + && fallbackCached.keyMetadata != null) { + LoggingUtils.log( + LOGGER_PROVIDER, + Level.FINE, + Collections.emptyMap(), + String.format( + "Agent Identity certificate config at %s is unavailable; falling back to cached" + + " certificate and key paths.", + certConfigPath)); + return new ResolvedCertAndKeyPaths( + fallbackCached.certMetadata.getPath(), + fallbackCached.keyMetadata.getPath(), + true, + fallbackCached.configMetadata); + } + + throw new IOException( + "Unable to find Agent Identity certificate config or file for bound token request" + + " after multiple retries. Token binding protection is failing. You can turn" + + " off this protection by setting " + + GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN + + " to false to fall back to unbound tokens.", + lastParseException); + } + + /** Searches for certificates at well-known locations. */ + private static ResolvedCertAndKeyPaths getWellKnownCertificatePathWithRetry( + final @Nullable CachedAgentIdentityInfo initialCached) throws IOException { + if (isMtlsExplicitlyDisabled()) { + initialStartupCompleted = true; + return new ResolvedCertAndKeyPaths(null, null, false); + } + + // Startup polling is only performed when GOOGLE_API_CERTIFICATE_CONFIG is set and points to the + // well-known directory (see getPathsFromConfigWithRetry). When GOOGLE_API_CERTIFICATE_CONFIG is + // unset, GKE mounts the Pod Certificate volume before the container starts (and on GCE an empty + // well-known directory may exist without any certificates ever being delivered), so we perform + // a single immediate check without startup polling. + // The GKE credential bundle is only used when no well-known gcloud certificate configuration + // file exists (it would otherwise be the transport's mTLS certificate source). + boolean gkeBundleAllowed = + !MtlsUtils.hasCertificateConfiguration(environmentProvider, propertyProvider); + String gkeBundlePath = gkeCredentialBundlePath; + if (gkeBundleAllowed && MtlsUtils.isGkeCredentialBundleReady(gkeBundlePath)) { + initialStartupCompleted = true; + // GKE combined bundle file contains both certificate chain and private key + return new ResolvedCertAndKeyPaths(gkeBundlePath, gkeBundlePath, false); + } + + if (!isMtlsExplicitlyEnabled()) { + // Without a config file or GOOGLE_API_USE_CLIENT_CERTIFICATE="true", only the GKE credential + // bundle can auto-enable mTLS. If a GKE credential was previously cached and the bundle is + // transiently unavailable (e.g. mid-rotation), return the cached paths so + // loadAndVerifyCredentials can fall back to the cached credential. + initialStartupCompleted = true; + CachedAgentIdentityInfo fallbackCached = + AgentIdentityCacheUtils.getLatestOrInitialCache(initialCached); + if (gkeBundleAllowed + && fallbackCached != null + && fallbackCached.canUseWellKnownFallback() + && fallbackCached.isGkeCredentialBundle) { + return new ResolvedCertAndKeyPaths( + fallbackCached.certMetadata.getPath(), fallbackCached.keyMetadata.getPath(), false); + } + return new ResolvedCertAndKeyPaths(null, null, false); + } + + String bundlePath = Paths.get(wellKnownDir, "credentialbundle.pem").toString(); + String certOnlyPath = Paths.get(wellKnownDir, "certificates.pem").toString(); + String keyOnlyPath = Paths.get(wellKnownDir, "private_key.pem").toString(); + + try { + if (AgentIdentityCacheUtils.checkExistsOrAccessDenied(Paths.get(bundlePath))) { + initialStartupCompleted = true; + // Combined bundle file contains both certificate chain and private key + return new ResolvedCertAndKeyPaths(bundlePath, bundlePath, false); + } + if (AgentIdentityCacheUtils.checkExistsOrAccessDenied(Paths.get(certOnlyPath)) + && AgentIdentityCacheUtils.checkExistsOrAccessDenied(Paths.get(keyOnlyPath))) { + initialStartupCompleted = true; + return new ResolvedCertAndKeyPaths(certOnlyPath, keyOnlyPath, false); + } + } catch (AccessDeniedException e) { + initialStartupCompleted = true; + LoggingUtils.log( + LOGGER_PROVIDER, + Level.FINE, + Collections.emptyMap(), + "Permission denied reading well-known certificates. Falling back to unbound token."); + return new ResolvedCertAndKeyPaths(null, null, false); + } catch (Exception e) { + LoggingUtils.log( + LOGGER_PROVIDER, + Level.FINE, + Collections.emptyMap(), + "Unexpected error checking well-known certificates: " + e.getMessage()); + } + initialStartupCompleted = true; + + CachedAgentIdentityInfo fallbackCached = + AgentIdentityCacheUtils.getLatestOrInitialCache(initialCached); + if (fallbackCached != null + && fallbackCached.canUseWellKnownFallback() + // A cached GKE credential is only reused while the GKE bundle is still allowed (no + // certificate configuration exists), matching the certificate the transport presents. + && (!fallbackCached.isGkeCredentialBundle || gkeBundleAllowed)) { + return new ResolvedCertAndKeyPaths( + fallbackCached.certMetadata.getPath(), fallbackCached.keyMetadata.getPath(), false); + } + + return new ResolvedCertAndKeyPaths(null, null, false); + } + + /** + * Determines if mTLS should be enabled based on environment variables and certificate presence. + * + *

    Note that {@code configExists} is required even when {@code certsPresent} is {@code true}: + * when {@code GOOGLE_API_USE_CLIENT_CERTIFICATE} is unset (Case 3), mTLS is only inferred if a + * workload config exists ({@code configExists == true}), whereas when {@code + * GOOGLE_API_USE_CLIENT_CERTIFICATE="true"} (Case 1), mTLS is enabled whenever {@code + * certsPresent == true} even from the well-known directory ({@code configExists == false}). + * + *

    Callers pass {@code configExists == true} for the GKE credential bundle as well (see {@link + * #GKE_CREDENTIAL_BUNDLE_FILE}), since its presence auto-enables mTLS on its own. + */ + static boolean shouldEnableMtls(final boolean certsPresent, final boolean configExists) + throws IOException { + String useClientCert = getUseClientCertificateEnv(); + + // Case 1: Explicitly enabled via environment variable + if ("true".equalsIgnoreCase(useClientCert)) { + if (certsPresent) { + // Certs are available, enable mTLS + return true; + } + if (configExists) { + // Config exists but files are missing - fail fast + throw new IOException( + "Certificate intent established via config, but cert files are missing."); + } + // Neither exist, do not enable + return false; + } + // Case 2: Explicitly disabled via environment variable (any non-empty value other than "true") + else if (!Strings.isNullOrEmpty(useClientCert)) { + if (certsPresent && !mtlsDisabledLogged) { + mtlsDisabledLogged = true; + // Log that we are ignoring present certs because it was explicitly disabled + LoggingUtils.log( + LOGGER_PROVIDER, + Level.FINE, + Collections.emptyMap(), + "Token binding protection is disabled because mTLS was explicitly disabled" + + " via GOOGLE_API_USE_CLIENT_CERTIFICATE."); + } + return false; + } + // Case 3: Environment variable is unset + else { + if (configExists) { + if (certsPresent) { + // Infer mTLS is enabled because workload config and certs are present + return true; + } + // Config exists but files are missing - fail fast + throw new IOException( + "Certificate intent inferred via config, but cert files are missing."); + } + // No workload config to infer mTLS from, do not enable + return false; + } + } + + /** Retrieves the bound token payload (certificate chain) if applicable. */ + static @Nullable String getBoundTokenPayload() throws IOException { + CertInfo info = getAgentIdentityCertInfo(); + return info != null ? info.getCertContent() : null; + } + + /** Extracts the certificate and private key paths from the JSON configuration file. */ + private static ResolvedCertAndKeyPaths extractPathsFromConfig(final String certConfigPath) + throws IOException { + try { + FileMetadata configMetadata = FileMetadata.of(certConfigPath); + try (InputStream stream = Files.newInputStream(Paths.get(certConfigPath))) { + JsonObjectParser parser = new JsonObjectParser(OAuth2Utils.JSON_FACTORY); + GenericJson config = + parser.parseAndClose(stream, StandardCharsets.UTF_8, GenericJson.class); + Object certConfigsObj = config.get("cert_configs"); + if (certConfigsObj instanceof Map) { + Map certConfigs = (Map) certConfigsObj; + Object workloadObj = certConfigs.get("workload"); + if (workloadObj instanceof Map) { + Map workload = (Map) workloadObj; + String certPath = + workload.get("cert_path") instanceof String + ? (String) workload.get("cert_path") + : null; + String keyPath = + workload.get("key_path") instanceof String + ? (String) workload.get("key_path") + : null; + if (Strings.isNullOrEmpty(certPath) || Strings.isNullOrEmpty(keyPath)) { + throw new IncompleteWorkloadConfigException( + "Workload certificate config is missing cert_path or key_path."); + } + return new ResolvedCertAndKeyPaths(certPath, keyPath, true, configMetadata); + } + } + return new ResolvedCertAndKeyPaths(null, null, false, configMetadata); + } + } catch (AccessDeniedException | IncompleteWorkloadConfigException e) { + throw e; + } catch (Exception e) { + throw new IOException("Failed to parse Agent Identity config JSON", e); + } + } + + /** Resets all static state and overrides for testing. */ + @VisibleForTesting + static void resetForTest() { + wellKnownDir = "/var/run/secrets/workload-spiffe-credentials/"; + gkeCredentialBundlePath = Paths.get(wellKnownDir, GKE_CREDENTIAL_BUNDLE_FILE).toString(); + environmentProvider = SystemEnvironmentProvider.getInstance(); + propertyProvider = SystemPropertyProvider.getInstance(); + timeService = Thread::sleep; + resetCachedState(); + } + + private static void resetCachedState() { + AgentIdentityCacheUtils.clearCachedAgentIdentityInfo(); + initialStartupCompleted = false; + mtlsDisabledLogged = false; + mtlsDisabledEvaluated = false; + } + + /** Returns the current well-known certificate directory path. */ + static String getWellKnownDir() { + return wellKnownDir; + } + + /** Sets the well-known certificate directory path for testing. */ + @VisibleForTesting + static void setWellKnownDir(final String dir) { + wellKnownDir = dir; + gkeCredentialBundlePath = Paths.get(dir, GKE_CREDENTIAL_BUNDLE_FILE).toString(); + resetCachedState(); + } + + /** Sets the environment variable provider for testing. */ + @VisibleForTesting + static void setEnvironmentProvider(final EnvironmentProvider provider) { + environmentProvider = provider; + resetCachedState(); + } + + /** + * Sets the system property provider for testing. Used to resolve the well-known gcloud + * certificate configuration file location. + */ + @VisibleForTesting + static void setPropertyProvider(final PropertyProvider provider) { + propertyProvider = provider; + resetCachedState(); + } + + /** Sets the time and sleep service for testing. */ + @VisibleForTesting + static void setTimeService(final TimeService service) { + timeService = service; + resetCachedState(); + } + + /** Returns whether the message for explicitly disabled mTLS has been logged. */ + @VisibleForTesting + static boolean isMtlsDisabledLogged() { + return mtlsDisabledLogged; + } +} diff --git a/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/CertificateIdentityPoolSubjectTokenSupplier.java b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/CertificateIdentityPoolSubjectTokenSupplier.java index 5b2ad1169ed0..dae2ddb57f7b 100644 --- a/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/CertificateIdentityPoolSubjectTokenSupplier.java +++ b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/CertificateIdentityPoolSubjectTokenSupplier.java @@ -51,7 +51,6 @@ import java.util.Base64; import java.util.List; import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.jspecify.annotations.NullMarked; /** @@ -66,9 +65,6 @@ public class CertificateIdentityPoolSubjectTokenSupplier private final IdentityPoolCredentialSource credentialSource; - private static final Pattern PEM_CERT_PATTERN = - Pattern.compile("-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", Pattern.DOTALL); - CertificateIdentityPoolSubjectTokenSupplier(IdentityPoolCredentialSource credentialSource) { this.credentialSource = checkNotNull(credentialSource, "credentialSource cannot be null"); // This check ensures that the credential source was intended for certificate usage. @@ -222,7 +218,8 @@ private void populateCertChainFromTrustChain( // elsewhere in the chain. if (encodedCurrentCert.equals(encodedLeafCert)) { throw new IllegalArgumentException( - "The leaf certificate should only appear at the beginning of the trust chain file, or be omitted entirely."); + "The leaf certificate should only appear at the beginning of the trust chain file, or" + + " be omitted entirely."); } // Add the current certificate to the chain. @@ -260,7 +257,7 @@ static List readTrustChain(String trustChainPath) // Split the file content into PEM certificate blocks. String content = new String(trustChainData, StandardCharsets.UTF_8); - Matcher matcher = PEM_CERT_PATTERN.matcher(content); + Matcher matcher = OAuth2Utils.PEM_CERT_PATTERN.matcher(content); while (matcher.find()) { String pemCertBlock = matcher.group(0); diff --git a/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/ComputeEngineCredentials.java b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/ComputeEngineCredentials.java index a3b87c0c6489..7495e3d08827 100644 --- a/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/ComputeEngineCredentials.java +++ b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/ComputeEngineCredentials.java @@ -34,11 +34,14 @@ import static com.google.common.base.MoreObjects.firstNonNull; import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpContent; import com.google.api.client.http.HttpHeaders; +import com.google.api.client.http.HttpMediaType; import com.google.api.client.http.HttpRequest; import com.google.api.client.http.HttpResponse; import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpStatusCodes; +import com.google.api.client.http.json.JsonHttpContent; import com.google.api.client.json.JsonObjectParser; import com.google.api.client.util.GenericData; import com.google.auth.CredentialTypeForMetrics; @@ -68,6 +71,7 @@ import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.logging.Level; import java.util.logging.Logger; @@ -77,7 +81,11 @@ /** * OAuth2 credentials representing the built-in service account for a Google Compute Engine VM. * - *

    Fetches access tokens from the Google Compute Engine metadata server. + *

    Fetches access tokens from the Google Compute Engine metadata server. When a workload + * certificate for an agent identity is available, requests certificate-bound access tokens and ID + * tokens by default (see {@code IdTokenProvider.Option.DISABLE_BOUND_ID_TOKEN} and the Google + * Auth Library guide). * *

    These credentials use the IAM API to sign data. See {@link #sign(byte[])} for more details. */ @@ -320,7 +328,8 @@ public String getUniverseDomain() throws IOException { private String getUniverseDomainFromMetadata() throws IOException { HttpResponse response = - getMetadataResponse(getUniverseDomainUrl(), RequestType.UNTRACKED, false); + getMetadataResponse( + buildMetadataRequest(getUniverseDomainUrl()), RequestType.UNTRACKED, false); int statusCode = response.getStatusCode(); if (statusCode == HttpStatusCodes.STATUS_CODE_NOT_FOUND) { return Credentials.GOOGLE_DEFAULT_UNIVERSE; @@ -379,7 +388,9 @@ private String getUniverseDomainFromMetadata() throws IOException { private @Nullable String getProjectIdFromMetadata() { try { - HttpResponse response = getMetadataResponse(getProjectIdUrl(), RequestType.UNTRACKED, false); + HttpResponse response = + getMetadataResponse( + buildMetadataRequest(getProjectIdUrl()), RequestType.UNTRACKED, false); int statusCode = response.getStatusCode(); if (statusCode == HttpStatusCodes.STATUS_CODE_NOT_FOUND) { LoggingUtils.log( @@ -421,10 +432,23 @@ private String getUniverseDomainFromMetadata() throws IOException { /** Refresh the access token by getting it from the GCE metadata server */ @Override public AccessToken refreshAccessToken() throws IOException { - HttpResponse response = - getMetadataResponse(createTokenUrlWithScopes(), RequestType.ACCESS_TOKEN_REQUEST, true); + String tokenUrl = createTokenUrlWithScopes(); + String boundTokenPayload = AgentIdentityUtils.getBoundTokenPayload(); + HttpRequest request = + boundTokenPayload != null + ? buildBoundTokenRequest(tokenUrl, boundTokenPayload) + : buildMetadataRequest(tokenUrl); + HttpResponse response = getMetadataResponse(request, RequestType.ACCESS_TOKEN_REQUEST, true); int statusCode = response.getStatusCode(); if (statusCode == HttpStatusCodes.STATUS_CODE_NOT_FOUND) { + if (boundTokenPayload != null) { + throw new IOException( + String.format( + "Error code %s trying to get bound security access token from Compute Engine" + + " metadata for the default service account. The Compute Engine metadata" + + " server endpoint does not support bound tokens.", + statusCode)); + } throw new IOException( String.format( "Error code %s trying to get security access token from Compute Engine metadata for" @@ -470,6 +494,8 @@ public AccessToken refreshAccessToken() throws IOException { *
    * IdTokenProvider.Option.FORMAT_FULL
    * IdTokenProvider.Option.LICENSES_TRUE
    + * IdTokenProvider.Option.DISABLE_BOUND_ID_TOKEN (request an ID token that is not bound to the + * agent identity certificate)
    * If no options are set, the defaults are "&format=standard&licenses=false" * @throws IOException if the attempt to get an IdToken failed * @return IdToken object which includes the raw id_token, JsonWebSignature @@ -477,7 +503,14 @@ public AccessToken refreshAccessToken() throws IOException { @Override public IdToken idTokenWithAudience( String targetAudience, @Nullable List options) throws IOException { + // Checked before getBoundTokenPayload() so an opted-out target skips the certificate lookup. + boolean bindIdToken = + options == null || !options.contains(IdTokenProvider.Option.DISABLE_BOUND_ID_TOKEN); + String boundTokenPayload = bindIdToken ? AgentIdentityUtils.getBoundTokenPayload() : null; GenericUrl documentUrl = new GenericUrl(getIdentityDocumentUrl()); + if (boundTokenPayload != null) { + documentUrl.set("format", "full"); + } if (options != null) { if (options.contains(IdTokenProvider.Option.FORMAT_FULL)) { documentUrl.set("format", "full"); @@ -489,10 +522,21 @@ public IdToken idTokenWithAudience( } } documentUrl.set("audience", targetAudience); - HttpResponse response = - getMetadataResponse(documentUrl.toString(), RequestType.ID_TOKEN_REQUEST, true); + HttpRequest request = + boundTokenPayload != null + ? buildBoundTokenRequest(documentUrl.toString(), boundTokenPayload) + : buildMetadataRequest(documentUrl.toString()); + HttpResponse response = getMetadataResponse(request, RequestType.ID_TOKEN_REQUEST, true); int statusCode = response.getStatusCode(); if (statusCode == HttpStatusCodes.STATUS_CODE_NOT_FOUND) { + if (boundTokenPayload != null) { + throw new IOException( + String.format( + "Error code %s trying to get bound identity token from Compute Engine metadata." + + " The Compute Engine metadata server endpoint does not support bound" + + " tokens.", + statusCode)); + } throw new IOException( String.format( "Error code %s trying to get identity token from" @@ -519,11 +563,25 @@ public IdToken idTokenWithAudience( return IdToken.create(rawToken); } + private HttpRequest buildMetadataRequest(String url) throws IOException { + return transportFactory.create().createRequestFactory().buildGetRequest(new GenericUrl(url)); + } + + private HttpRequest buildBoundTokenRequest(String url, String boundTokenPayload) + throws IOException { + Map payload = Collections.singletonMap("certificate_chain", boundTokenPayload); + HttpContent content = + new JsonHttpContent(OAuth2Utils.JSON_FACTORY, payload) + .setMediaType(new HttpMediaType("application/json")); + return transportFactory + .create() + .createRequestFactory() + .buildPostRequest(new GenericUrl(url), content); + } + private HttpResponse getMetadataResponse( - String url, RequestType requestType, boolean shouldSendMetricsHeader) throws IOException { - GenericUrl genericUrl = new GenericUrl(url); - HttpRequest request = - transportFactory.create().createRequestFactory().buildGetRequest(genericUrl); + HttpRequest request, RequestType requestType, boolean shouldSendMetricsHeader) + throws IOException { // Disable automatic logging by google-http-java-client to prevent leakage of sensitive tokens. // Client Library Debug Logging via LoggingUtils is used instead where appropriate. request.setLoggingEnabled(false); @@ -840,7 +898,8 @@ public byte[] sign(byte[] toSign) { private String getDefaultServiceAccount() throws IOException { HttpResponse response = - getMetadataResponse(getDefaultServiceAccountUrl(), RequestType.UNTRACKED, false); + getMetadataResponse( + buildMetadataRequest(getDefaultServiceAccountUrl()), RequestType.UNTRACKED, false); int statusCode = response.getStatusCode(); if (statusCode == HttpStatusCodes.STATUS_CODE_NOT_FOUND) { throw new IOException( diff --git a/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/IdTokenProvider.java b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/IdTokenProvider.java index d1dc1112b85f..e1673f19d3da 100644 --- a/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/IdTokenProvider.java +++ b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/IdTokenProvider.java @@ -48,6 +48,7 @@ public interface IdTokenProvider { *

    * *
    @@ -60,7 +61,28 @@ public interface IdTokenProvider { public enum Option { FORMAT_FULL("formatFull"), LICENSES_TRUE("licensesTrue"), - INCLUDE_EMAIL("includeEmail"); + INCLUDE_EMAIL("includeEmail"), + /** + * Requests an ID token that is not bound to the workload's agent identity certificate. + * + *

    This option is only supported by {@link ComputeEngineCredentials}; other {@link + * IdTokenProvider} implementations do not request bound ID tokens and ignore this option. + * + *

    When an agent identity certificate is available, {@link ComputeEngineCredentials} requests + * certificate-bound ID tokens by default. A bound ID token is only accepted by targets that + * authenticate the caller over mTLS with the same certificate. Pass this option for targets + * that are called over standard (non-mTLS) HTTPS, for example a Cloud Run service reached + * through its {@code *.run.app} URL or a custom domain. + * + *

    Unlike the {@code GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN=false} environment variable, which + * globally disables certificate-bound access tokens and ID tokens across the entire process, + * this option applies per call (or per {@link IdTokenCredentials} instance) and leaves access + * tokens and other ID token requests bound. If this option is not set, the library decides + * whether to bind the ID token. Currently, it is bound whenever an agent identity certificate + * is available and token binding is not globally disabled by {@code + * GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN=false}. + */ + DISABLE_BOUND_ID_TOKEN("disableBoundIdToken"); private final String option; diff --git a/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/OAuth2Utils.java b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/OAuth2Utils.java index f740dd980e73..6d068c173dfb 100644 --- a/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/OAuth2Utils.java +++ b/google-auth-library-java/oauth2_http/java/com/google/auth/oauth2/OAuth2Utils.java @@ -68,6 +68,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Pattern; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; @@ -81,6 +82,9 @@ enum Pkcs8Algorithm { EC } + static final Pattern PEM_CERT_PATTERN = + Pattern.compile("-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", Pattern.DOTALL); + static final String SIGNATURE_ALGORITHM = "SHA256withRSA"; public static final String TOKEN_TYPE_ACCESS_TOKEN = diff --git a/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/MtlsKeyStoreUtilsTest.java b/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/MtlsKeyStoreUtilsTest.java new file mode 100644 index 000000000000..5f1e8fca0545 --- /dev/null +++ b/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/MtlsKeyStoreUtilsTest.java @@ -0,0 +1,345 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.google.auth.mtls; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.api.client.util.SecurityUtils; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.Signature; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.security.spec.InvalidKeySpecException; +import java.util.Collections; +import org.junit.jupiter.api.Test; + +class MtlsKeyStoreUtilsTest { + + // Leaf (CN=Test Leaf) followed by the intermediate that signed it (CN=Test Intermediate CA). + static final String CHAIN_CERT_PATH = "testresources/mtls/test_chain_cert.pem"; + // PKCS#8 private key for the leaf in CHAIN_CERT_PATH. + static final String CHAIN_KEY_PATH = "testresources/mtls/test_chain_key.pem"; + private static final String SINGLE_CERT_PATH = "testresources/mtls/test_cert.pem"; + private static final String SINGLE_KEY_PATH = "testresources/mtls/test_key.pem"; + + @Test + void createMtlsKeyStore_certificatesFirst_keepsFullChainInOrder() throws Exception { + KeyStore keyStore = + MtlsKeyStoreUtils.createMtlsKeyStore(stream(read(CHAIN_CERT_PATH), read(CHAIN_KEY_PATH))); + + assertLeafThenIntermediate(keyStore); + } + + @Test + void createMtlsKeyStore_keyFirst_keepsFullChainInOrder() throws Exception { + KeyStore keyStore = + MtlsKeyStoreUtils.createMtlsKeyStore(stream(read(CHAIN_KEY_PATH), read(CHAIN_CERT_PATH))); + + assertLeafThenIntermediate(keyStore); + } + + @Test + void createMtlsKeyStore_threeCertificates_keepsFullChainInOrder() throws Exception { + KeyStore keyStore = + MtlsKeyStoreUtils.createMtlsKeyStore( + stream(read(CHAIN_CERT_PATH), read(SINGLE_CERT_PATH), read(CHAIN_KEY_PATH))); + + Certificate[] chain = keyStore.getCertificateChain(onlyAlias(keyStore)); + assertEquals(3, chain.length); + assertEquals("CN=Test Leaf", subject(chain[0])); + assertEquals("CN=Test Intermediate CA", subject(chain[1])); + assertEquals( + subject( + SecurityUtils.createMtlsKeyStore(stream(read(SINGLE_CERT_PATH), read(SINGLE_KEY_PATH))) + .getCertificate("alias")), + subject(chain[2])); + } + + @Test + void createMtlsKeyStore_duplicateCertificates_deduplicated() throws Exception { + // Simulates X509Provider concatenating cert_path and key_path when both point to the same + // combined PEM bundle. + String combinedBundle = read(CHAIN_CERT_PATH) + "\n" + read(CHAIN_KEY_PATH); + KeyStore keyStore = + MtlsKeyStoreUtils.createMtlsKeyStore(stream(combinedBundle, combinedBundle)); + + assertLeafThenIntermediate(keyStore); + } + + @Test + void upstreamCreateMtlsKeyStore_keepsOnlyLeaf() throws Exception { + // Documents why MtlsKeyStoreUtils exists. If google-http-client starts keeping the full chain, + // this fails and the helper can be replaced by the upstream method again. + KeyStore keyStore = + SecurityUtils.createMtlsKeyStore(stream(read(CHAIN_CERT_PATH), read(CHAIN_KEY_PATH))); + + assertEquals(1, keyStore.getCertificateChain(onlyAlias(keyStore)).length); + } + + @Test + void createMtlsKeyStore_privateKeyMatchesLeaf() throws Exception { + KeyStore keyStore = + MtlsKeyStoreUtils.createMtlsKeyStore(stream(read(CHAIN_CERT_PATH), read(CHAIN_KEY_PATH))); + String alias = onlyAlias(keyStore); + PrivateKey key = (PrivateKey) keyStore.getKey(alias, new char[] {}); + Certificate leaf = keyStore.getCertificateChain(alias)[0]; + + byte[] data = "chain".getBytes(UTF_8); + Signature signer = Signature.getInstance("SHA256withECDSA"); + signer.initSign(key); + signer.update(data); + byte[] signature = signer.sign(); + Signature verifier = Signature.getInstance("SHA256withECDSA"); + verifier.initVerify(leaf.getPublicKey()); + verifier.update(data); + assertTrue(verifier.verify(signature)); + } + + @Test + void createMtlsKeyStore_singleCertificate_sameAsBefore() throws Exception { + KeyStore keyStore = + MtlsKeyStoreUtils.createMtlsKeyStore(stream(read(SINGLE_CERT_PATH), read(SINGLE_KEY_PATH))); + KeyStore expected = + SecurityUtils.createMtlsKeyStore(stream(read(SINGLE_CERT_PATH), read(SINGLE_KEY_PATH))); + + String alias = onlyAlias(keyStore); + assertEquals(Collections.list(expected.aliases()), Collections.list(keyStore.aliases())); + assertEquals(expected.getType(), keyStore.getType()); + Certificate[] chain = keyStore.getCertificateChain(alias); + assertEquals(1, chain.length); + assertArrayEquals(expected.getCertificate(alias).getEncoded(), chain[0].getEncoded()); + assertArrayEquals( + expected.getKey(alias, new char[] {}).getEncoded(), + keyStore.getKey(alias, new char[] {}).getEncoded()); + } + + @Test + void createMtlsKeyStore_ignoresOtherSections() throws Exception { + String other = "-----BEGIN EC PARAMETERS-----\nBggqhkjOPQMBBw==\n-----END EC PARAMETERS-----\n"; + KeyStore keyStore = + MtlsKeyStoreUtils.createMtlsKeyStore( + stream(other, read(CHAIN_CERT_PATH), other, read(CHAIN_KEY_PATH))); + + assertEquals(2, keyStore.getCertificateChain(onlyAlias(keyStore)).length); + } + + @Test + void createMtlsKeyStore_keysBeforeCertificate_usesLastKeyLikeUpstream() throws Exception { + // RSA key, then the leaf's EC key, then the certificates: the key just before the first + // certificate is used. + String input = + String.join("\n", read(SINGLE_KEY_PATH), read(CHAIN_KEY_PATH), read(CHAIN_CERT_PATH)); + + KeyStore keyStore = MtlsKeyStoreUtils.createMtlsKeyStore(stream(input)); + KeyStore upstream = SecurityUtils.createMtlsKeyStore(stream(input)); + + byte[] expectedKeyBytes = expectedChainKeyEncoded(); + assertArrayEquals( + expectedKeyBytes, keyStore.getKey(onlyAlias(keyStore), new char[] {}).getEncoded()); + assertArrayEquals( + expectedKeyBytes, upstream.getKey(onlyAlias(upstream), new char[] {}).getEncoded()); + assertLeafThenIntermediate(keyStore); + } + + @Test + void createMtlsKeyStore_keysBeforeCertificate_lastKeyDoesNotMatch_throwsLikeUpstream() + throws Exception { + // The leaf's EC key, then an RSA key, then the certificates: the RSA key is chosen and cannot + // be read as an EC key, in both implementations. + String input = + String.join("\n", read(CHAIN_KEY_PATH), read(SINGLE_KEY_PATH), read(CHAIN_CERT_PATH)); + + assertThrows( + InvalidKeySpecException.class, () -> MtlsKeyStoreUtils.createMtlsKeyStore(stream(input))); + assertThrows( + InvalidKeySpecException.class, () -> SecurityUtils.createMtlsKeyStore(stream(input))); + } + + @Test + void createMtlsKeyStore_keysAfterCertificate_usesFirstKeyLikeUpstream() throws Exception { + String input = + String.join("\n", read(CHAIN_CERT_PATH), read(CHAIN_KEY_PATH), read(SINGLE_KEY_PATH)); + + KeyStore keyStore = MtlsKeyStoreUtils.createMtlsKeyStore(stream(input)); + KeyStore upstream = SecurityUtils.createMtlsKeyStore(stream(input)); + + byte[] expectedKeyBytes = expectedChainKeyEncoded(); + assertArrayEquals( + expectedKeyBytes, keyStore.getKey(onlyAlias(keyStore), new char[] {}).getEncoded()); + assertArrayEquals( + expectedKeyBytes, upstream.getKey(onlyAlias(upstream), new char[] {}).getEncoded()); + assertLeafThenIntermediate(keyStore); + } + + @Test + void createMtlsKeyStore_malformedTrailingSection_ignored() throws Exception { + // A section with no end tag after a usable certificate and key. The upstream method stopped + // reading before reaching it, so it must not cause a failure now. Putting the intermediate + // after the key also checks that reading continues past the first certificate and key. + String[] leafAndIntermediate = + read(CHAIN_CERT_PATH).split("(?<=-----END CERTIFICATE-----)\r?\n"); + KeyStore keyStore = + MtlsKeyStoreUtils.createMtlsKeyStore( + stream( + leafAndIntermediate[0], + read(CHAIN_KEY_PATH), + leafAndIntermediate[1], + "-----BEGIN CERTIFICATE-----\nAAAA")); + + assertLeafThenIntermediate(keyStore); + } + + @Test + void createMtlsKeyStore_malformedSectionAfterOnlyCertificate_throws() throws Exception { + assertThrows( + IllegalArgumentException.class, + () -> + MtlsKeyStoreUtils.createMtlsKeyStore( + stream(read(CHAIN_CERT_PATH), "-----BEGIN PRIVATE KEY-----\nAAAA"))); + } + + @Test + void createMtlsKeyStore_malformedSectionAfterOnlyKey_throws() throws Exception { + assertThrows( + IllegalArgumentException.class, + () -> + MtlsKeyStoreUtils.createMtlsKeyStore( + stream(read(CHAIN_KEY_PATH), "-----BEGIN CERTIFICATE-----\nAAAA"))); + } + + @Test + void createMtlsKeyStore_ioExceptionAfterCertAndKey_throws() throws Exception { + String[] leafAndIntermediate = + read(CHAIN_CERT_PATH).split("(?<=-----END CERTIFICATE-----)\r?\n"); + byte[] validPrefix = (read(CHAIN_KEY_PATH) + "\n" + leafAndIntermediate[0]).getBytes(UTF_8); + InputStream failingStream = + new FilterInputStream(new ByteArrayInputStream(validPrefix)) { + @Override + public int read(byte[] b, int off, int len) throws IOException { + int n = super.read(b, off, len); + if (n == -1) { + throw new IOException("disk read error while reading intermediate"); + } + return n; + } + }; + + assertThrows(IOException.class, () -> MtlsKeyStoreUtils.createMtlsKeyStore(failingStream)); + } + + @Test + void createMtlsKeyStore_unparsableExtraCertificate_leftOut() throws Exception { + String[] leafAndIntermediate = + read(CHAIN_CERT_PATH).split("(?<=-----END CERTIFICATE-----)\r?\n"); + String bad = "-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----"; + String empty = "-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----"; + KeyStore keyStore = + MtlsKeyStoreUtils.createMtlsKeyStore( + stream( + leafAndIntermediate[0], bad, empty, leafAndIntermediate[1], read(CHAIN_KEY_PATH))); + + assertLeafThenIntermediate(keyStore); + } + + @Test + void createMtlsKeyStore_unparsableLeaf_throws() throws Exception { + String bad = "-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----"; + assertThrows( + CertificateException.class, + () -> + MtlsKeyStoreUtils.createMtlsKeyStore( + stream(bad, read(CHAIN_CERT_PATH), read(CHAIN_KEY_PATH)))); + } + + @Test + void createMtlsKeyStore_missingCertificate_throws() throws Exception { + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> MtlsKeyStoreUtils.createMtlsKeyStore(stream(read(CHAIN_KEY_PATH)))); + assertEquals("certificate is missing from certAndKey string", e.getMessage()); + } + + @Test + void createMtlsKeyStore_missingPrivateKey_throws() throws Exception { + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> MtlsKeyStoreUtils.createMtlsKeyStore(stream(read(CHAIN_CERT_PATH)))); + assertEquals("private key is missing from certAndKey string", e.getMessage()); + } + + static String read(String path) throws IOException { + return new String(Files.readAllBytes(new File(path).toPath()), UTF_8); + } + + /** Asserts the key store's single entry holds the test leaf followed by its intermediate. */ + static void assertLeafThenIntermediate(KeyStore keyStore) throws Exception { + Certificate[] chain = keyStore.getCertificateChain(onlyAlias(keyStore)); + assertEquals(2, chain.length); + assertEquals("CN=Test Leaf", subject(chain[0])); + assertEquals("CN=Test Intermediate CA", subject(chain[1])); + } + + private static ByteArrayInputStream stream(String... parts) { + return new ByteArrayInputStream(String.join("\n", parts).getBytes(UTF_8)); + } + + private static String onlyAlias(KeyStore keyStore) throws Exception { + assertEquals(1, keyStore.size()); + String alias = keyStore.aliases().nextElement(); + assertNotNull(keyStore.getCertificateChain(alias)); + return alias; + } + + private static byte[] expectedChainKeyEncoded() throws Exception { + KeyStore expected = + SecurityUtils.createMtlsKeyStore(stream(read(CHAIN_CERT_PATH), read(CHAIN_KEY_PATH))); + return expected.getKey(onlyAlias(expected), new char[] {}).getEncoded(); + } + + private static String subject(Certificate certificate) { + return ((X509Certificate) certificate).getSubjectX500Principal().getName(); + } +} diff --git a/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/MtlsUtilsTest.java b/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/MtlsUtilsTest.java index f3fdf05a4c32..34395805b5e9 100644 --- a/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/MtlsUtilsTest.java +++ b/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/MtlsUtilsTest.java @@ -39,6 +39,9 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -46,6 +49,18 @@ class MtlsUtilsTest { @TempDir Path tempDir; + @BeforeEach + void setUp() { + // Never consult the real GKE well-known path from tests. + MtlsUtils.setGkeCredentialBundlePathForTesting( + tempDir.resolve("no-gke-bundle").resolve("bundle.pem").toString()); + } + + @AfterEach + void tearDown() { + MtlsUtils.setGkeCredentialBundlePathForTesting(null); + } + @Test void getCertificatePath_succeeds() throws IOException { Path configFile = tempDir.resolve("config.json"); @@ -101,6 +116,21 @@ public String getProperty(String name, String def) { () -> MtlsUtils.getCertificatePath(envProvider, propProvider, configFile.toString())); } + @Test + void getCertificatePath_ecpOnlyConfig_throwsCertificateSourceUnavailableException() + throws IOException { + Path configFile = tempDir.resolve("ecp_config.json"); + Files.write( + configFile, "{\"cert_configs\":{\"enterprise_certificates\":{\"libs\":[]}}}".getBytes()); + + EnvironmentProvider envProvider = name -> null; + PropertyProvider propProvider = (name, def) -> def; + + assertThrows( + CertificateSourceUnavailableException.class, + () -> MtlsUtils.getCertificatePath(envProvider, propProvider, configFile.toString())); + } + @Test void getWorkloadCertificateConfiguration_overridePath() throws IOException { Path configFile = tempDir.resolve("custom_config.json"); @@ -243,4 +273,616 @@ public String getProperty(String name, String def) { assertEquals("APPDATA environment variable is not set on Windows.", exception.getMessage()); } + + @Test + void + useMtlsClientCertificate_trueWithNoCertsOnDisk_returnsTrueWhileWorkloadCertPathReturnsNull() { + EnvironmentProvider envProvider = + name -> "GOOGLE_API_USE_CLIENT_CERTIFICATE".equals(name) ? "true" : null; + PropertyProvider propProvider = + (name, def) -> { + if ("user.home".equals(name)) return tempDir.toString(); + if ("os.name".equals(name)) return "Linux"; + return def; + }; + + assertTrue(MtlsUtils.useMtlsClientCertificate(envProvider, propProvider)); + assertNull(MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + } + + @Test + void useMtlsClientCertificate_trueWithEcpOnlyConfig_returnsTrueAndWorkloadCertPathReturnsNull() + throws IOException { + Path configFile = tempDir.resolve("ecp_config.json"); + Files.write(configFile, "{\"cert_configs\":{\"enterprise_certificates\":{}}}".getBytes()); + + EnvironmentProvider envProvider = + name -> { + if ("GOOGLE_API_USE_CLIENT_CERTIFICATE".equals(name)) return "true"; + if ("GOOGLE_API_CERTIFICATE_CONFIG".equals(name)) return configFile.toString(); + return null; + }; + PropertyProvider propProvider = (name, def) -> def; + + assertTrue(MtlsUtils.useMtlsClientCertificate(envProvider, propProvider)); + assertNull(MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + } + + @Test + void useMtlsClientCertificate_false_returnsFalse() { + EnvironmentProvider envProvider = + name -> "GOOGLE_API_USE_CLIENT_CERTIFICATE".equals(name) ? "false" : null; + PropertyProvider propProvider = (name, def) -> def; + + assertFalse(MtlsUtils.useMtlsClientCertificate(envProvider, propProvider)); + assertNull(MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + } + + @Test + void useMtlsClientCertificate_falseEvenWhenWorkloadCertsExist_returnsFalse() throws IOException { + Path certFile = tempDir.resolve("cert.pem"); + Path keyFile = tempDir.resolve("key.pem"); + Files.write(certFile, "dummy cert".getBytes()); + Files.write(keyFile, "dummy key".getBytes()); + + Path configFile = tempDir.resolve("config.json"); + String configJson = + String.format( + "{\"cert_configs\":{\"workload\":{\"cert_path\":\"%s\",\"key_path\":\"%s\"}}}", + certFile.toString().replace("\\", "\\\\"), keyFile.toString().replace("\\", "\\\\")); + Files.write(configFile, configJson.getBytes()); + + EnvironmentProvider envProvider = + name -> { + if ("GOOGLE_API_USE_CLIENT_CERTIFICATE".equals(name)) return "false"; + if ("GOOGLE_API_CERTIFICATE_CONFIG".equals(name)) return configFile.toString(); + return null; + }; + PropertyProvider propProvider = (name, def) -> def; + + assertFalse(MtlsUtils.useMtlsClientCertificate(envProvider, propProvider)); + assertNull(MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + } + + @Test + void useMtlsClientCertificate_unsetWithNoCertsOnDisk_returnsFalse() { + EnvironmentProvider envProvider = name -> null; + PropertyProvider propProvider = + (name, def) -> { + if ("user.home".equals(name)) return tempDir.toString(); + if ("os.name".equals(name)) return "Linux"; + return def; + }; + + assertFalse(MtlsUtils.useMtlsClientCertificate(envProvider, propProvider)); + assertNull(MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + } + + // --- Explicit GOOGLE_API_CERTIFICATE_CONFIG Tests (Fail Closed) --- + + @Test + void getWorkloadCertPath_explicitConfigMissing_throwsIllegalStateException() { + EnvironmentProvider envProvider = + name -> "GOOGLE_API_CERTIFICATE_CONFIG".equals(name) ? "/nonexistent/config.json" : null; + PropertyProvider propProvider = (name, def) -> def; + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + assertTrue( + exception + .getMessage() + .contains( + "specified via GOOGLE_API_CERTIFICATE_CONFIG at '/nonexistent/config.json' does not" + + " exist")); + } + + @Test + void getWorkloadCertPath_explicitConfigIsDirectory_throwsIllegalStateException() + throws IOException { + Path configDir = tempDir.resolve("config_dir"); + Files.createDirectory(configDir); + + EnvironmentProvider envProvider = + name -> "GOOGLE_API_CERTIFICATE_CONFIG".equals(name) ? configDir.toString() : null; + PropertyProvider propProvider = (name, def) -> def; + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + assertTrue( + exception + .getMessage() + .contains( + "Failed to read certificate configuration file specified via" + + " GOOGLE_API_CERTIFICATE_CONFIG")); + } + + @Test + void getWorkloadCertPath_explicitConfigUnreadable_throwsIllegalStateException() + throws IOException { + Path configFile = tempDir.resolve("unreadable_config.json"); + Files.write(configFile, "{}".getBytes()); + File file = configFile.toFile(); + if (file.setReadable(false)) { + try { + Assumptions.assumeFalse(file.canRead(), "Current user can read any file (e.g. root)"); + EnvironmentProvider envProvider = + name -> "GOOGLE_API_CERTIFICATE_CONFIG".equals(name) ? configFile.toString() : null; + PropertyProvider propProvider = (name, def) -> def; + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + assertTrue( + exception + .getMessage() + .contains( + "Failed to read certificate configuration file specified via" + + " GOOGLE_API_CERTIFICATE_CONFIG")); + } finally { + file.setReadable(true); + } + } + } + + @Test + void getWorkloadCertPath_explicitConfigMalformedJson_throwsIllegalStateException() + throws IOException { + Path configFile = tempDir.resolve("malformed.json"); + Files.write(configFile, "{ invalid json".getBytes()); + + EnvironmentProvider envProvider = + name -> "GOOGLE_API_CERTIFICATE_CONFIG".equals(name) ? configFile.toString() : null; + PropertyProvider propProvider = (name, def) -> def; + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + assertTrue( + exception + .getMessage() + .contains( + "specified via GOOGLE_API_CERTIFICATE_CONFIG at '" + + configFile.toString() + + "' is malformed")); + } + + @Test + void getWorkloadCertPath_explicitConfigOnlyEcp_returnsNullSafely() throws IOException { + Path configFile = tempDir.resolve("ecp_config.json"); + Files.write(configFile, "{\"cert_configs\":{\"enterprise_certificates\":{}}}".getBytes()); + + EnvironmentProvider envProvider = + name -> "GOOGLE_API_CERTIFICATE_CONFIG".equals(name) ? configFile.toString() : null; + PropertyProvider propProvider = (name, def) -> def; + + assertNull(MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + } + + @Test + void getWorkloadCertPath_explicitConfigCertFileMissing_throwsIllegalStateException() + throws IOException { + Path keyFile = tempDir.resolve("key.pem"); + Files.write(keyFile, "dummy key".getBytes()); + + Path configFile = tempDir.resolve("config.json"); + String configJson = + String.format( + "{\"cert_configs\":{\"workload\":{\"cert_path\":\"/nonexistent/cert.pem\",\"key_path\":\"%s\"}}}", + keyFile.toString().replace("\\", "\\\\")); + Files.write(configFile, configJson.getBytes()); + + EnvironmentProvider envProvider = + name -> "GOOGLE_API_CERTIFICATE_CONFIG".equals(name) ? configFile.toString() : null; + PropertyProvider propProvider = (name, def) -> def; + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + assertTrue(exception.getMessage().contains("Failed to read certificate/key file")); + assertTrue( + exception + .getMessage() + .contains("referenced by configuration '" + configFile.toString() + "'")); + } + + @Test + void getWorkloadCertPath_explicitConfigCertFileIsDirectory_throwsIllegalStateException() + throws IOException { + Path certDir = tempDir.resolve("cert_dir"); + Files.createDirectory(certDir); + Path keyFile = tempDir.resolve("key.pem"); + Files.write(keyFile, "dummy key".getBytes()); + + Path configFile = tempDir.resolve("config.json"); + String configJson = + String.format( + "{\"cert_configs\":{\"workload\":{\"cert_path\":\"%s\",\"key_path\":\"%s\"}}}", + certDir.toString().replace("\\", "\\\\"), keyFile.toString().replace("\\", "\\\\")); + Files.write(configFile, configJson.getBytes()); + + EnvironmentProvider envProvider = + name -> "GOOGLE_API_CERTIFICATE_CONFIG".equals(name) ? configFile.toString() : null; + PropertyProvider propProvider = (name, def) -> def; + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + assertTrue(exception.getMessage().contains("Failed to read certificate/key file")); + } + + @Test + void getWorkloadCertPath_explicitConfigKeyFileMissing_throwsIllegalStateException() + throws IOException { + Path certFile = tempDir.resolve("cert.pem"); + Files.write(certFile, "dummy cert".getBytes()); + + Path configFile = tempDir.resolve("config.json"); + String configJson = + String.format( + "{\"cert_configs\":{\"workload\":{\"cert_path\":\"%s\",\"key_path\":\"/nonexistent/key.pem\"}}}", + certFile.toString().replace("\\", "\\\\")); + Files.write(configFile, configJson.getBytes()); + + EnvironmentProvider envProvider = + name -> "GOOGLE_API_CERTIFICATE_CONFIG".equals(name) ? configFile.toString() : null; + PropertyProvider propProvider = (name, def) -> def; + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + assertTrue(exception.getMessage().contains("Failed to read certificate/key file")); + assertTrue( + exception + .getMessage() + .contains("referenced by configuration '" + configFile.toString() + "'")); + } + + @Test + void getWorkloadCertPath_explicitConfigValid_returnsCertPath() throws IOException { + Path certFile = tempDir.resolve("cert.pem"); + Path keyFile = tempDir.resolve("key.pem"); + Files.write(certFile, "dummy cert".getBytes()); + Files.write(keyFile, "dummy key".getBytes()); + + Path configFile = tempDir.resolve("config.json"); + String configJson = + String.format( + "{\"cert_configs\":{\"workload\":{\"cert_path\":\"%s\",\"key_path\":\"%s\"}}}", + certFile.toString().replace("\\", "\\\\"), keyFile.toString().replace("\\", "\\\\")); + Files.write(configFile, configJson.getBytes()); + + EnvironmentProvider envProvider = + name -> "GOOGLE_API_CERTIFICATE_CONFIG".equals(name) ? configFile.toString() : null; + PropertyProvider propProvider = (name, def) -> def; + + assertTrue(MtlsUtils.useMtlsClientCertificate(envProvider, propProvider)); + assertEquals(certFile.toString(), MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + } + + // --- Implicit / Default gcloud Config Tests --- + + @Test + void getWorkloadCertPath_defaultConfigMissing_returnsNullSafely() { + EnvironmentProvider envProvider = name -> null; + PropertyProvider propProvider = + (name, def) -> { + if ("user.home".equals(name)) return tempDir.toString(); + if ("os.name".equals(name)) return "Linux"; + return def; + }; + + assertNull(MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + } + + @Test + void getWorkloadCertPath_defaultConfigIsDirectory_throwsIllegalStateException() + throws IOException { + Path gcloudDir = tempDir.resolve(".config/gcloud"); + Files.createDirectories(gcloudDir); + Path defaultConfigFile = gcloudDir.resolve("certificate_config.json"); + Files.createDirectory(defaultConfigFile); + + EnvironmentProvider envProvider = name -> null; + PropertyProvider propProvider = + (name, def) -> { + if ("user.home".equals(name)) return tempDir.toString(); + if ("os.name".equals(name)) return "Linux"; + return def; + }; + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + assertTrue( + exception + .getMessage() + .contains( + "Default certificate configuration file at '" + + defaultConfigFile.toFile().getAbsolutePath() + + "' exists but could not be read")); + } + + @Test + void getWorkloadCertPath_defaultConfigMalformedJson_throwsIllegalStateException() + throws IOException { + Path gcloudDir = tempDir.resolve(".config/gcloud"); + Files.createDirectories(gcloudDir); + Path defaultConfigFile = gcloudDir.resolve("certificate_config.json"); + Files.write(defaultConfigFile, "{ malformed json".getBytes()); + + EnvironmentProvider envProvider = name -> null; + PropertyProvider propProvider = + (name, def) -> { + if ("user.home".equals(name)) return tempDir.toString(); + if ("os.name".equals(name)) return "Linux"; + return def; + }; + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + assertTrue( + exception + .getMessage() + .contains( + "Default certificate configuration file at '" + + defaultConfigFile.toFile().getAbsolutePath() + + "' is malformed")); + } + + @Test + void getWorkloadCertPath_defaultConfigOnlyEcp_returnsNullSafely() throws IOException { + Path gcloudDir = tempDir.resolve(".config/gcloud"); + Files.createDirectories(gcloudDir); + Path defaultConfigFile = gcloudDir.resolve("certificate_config.json"); + Files.write( + defaultConfigFile, + "{\"cert_configs\":{\"enterprise_certificates\":{\"libs\":[]}}}".getBytes()); + + EnvironmentProvider envProvider = name -> null; + PropertyProvider propProvider = + (name, def) -> { + if ("user.home".equals(name)) return tempDir.toString(); + if ("os.name".equals(name)) return "Linux"; + return def; + }; + + assertNull(MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + } + + @Test + void getWorkloadCertPath_defaultConfigCertFileMissing_throwsIllegalStateException() + throws IOException { + Path keyFile = tempDir.resolve("key.pem"); + Files.write(keyFile, "dummy key".getBytes()); + + Path gcloudDir = tempDir.resolve(".config/gcloud"); + Files.createDirectories(gcloudDir); + Path defaultConfigFile = gcloudDir.resolve("certificate_config.json"); + String configJson = + String.format( + "{\"cert_configs\":{\"workload\":{\"cert_path\":\"/nonexistent/cert.pem\",\"key_path\":\"%s\"}}}", + keyFile.toString().replace("\\", "\\\\")); + Files.write(defaultConfigFile, configJson.getBytes()); + + EnvironmentProvider envProvider = name -> null; + PropertyProvider propProvider = + (name, def) -> { + if ("user.home".equals(name)) return tempDir.toString(); + if ("os.name".equals(name)) return "Linux"; + return def; + }; + + IllegalStateException exception = + assertThrows( + IllegalStateException.class, + () -> MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + assertTrue(exception.getMessage().contains("Failed to read certificate/key file")); + assertTrue( + exception + .getMessage() + .contains( + "referenced by default configuration '" + + defaultConfigFile.toFile().getAbsolutePath() + + "'")); + } + + @Test + void getWorkloadCertPath_defaultConfigValid_returnsCertPath() throws IOException { + Path certFile = tempDir.resolve("cert.pem"); + Path keyFile = tempDir.resolve("key.pem"); + Files.write(certFile, "dummy cert".getBytes()); + Files.write(keyFile, "dummy key".getBytes()); + + Path gcloudDir = tempDir.resolve(".config/gcloud"); + Files.createDirectories(gcloudDir); + Path defaultConfigFile = gcloudDir.resolve("certificate_config.json"); + String configJson = + String.format( + "{\"cert_configs\":{\"workload\":{\"cert_path\":\"%s\",\"key_path\":\"%s\"}}}", + certFile.toString().replace("\\", "\\\\"), keyFile.toString().replace("\\", "\\\\")); + Files.write(defaultConfigFile, configJson.getBytes()); + + EnvironmentProvider envProvider = name -> null; + PropertyProvider propProvider = + (name, def) -> { + if ("user.home".equals(name)) return tempDir.toString(); + if ("os.name".equals(name)) return "Linux"; + return def; + }; + + assertTrue(MtlsUtils.useMtlsClientCertificate(envProvider, propProvider)); + assertEquals(certFile.toString(), MtlsUtils.getWorkloadCertPath(envProvider, propProvider)); + } + + // --- General Helpers & Stubs Tests --- + + @Test + void getCertificateFingerprint_validFile_returnsSha256() throws IOException { + Path file = tempDir.resolve("test.crt"); + Files.write(file, "hello world".getBytes()); + + String fingerprint = MtlsUtils.getCertificateFingerprint(file.toString()); + assertNotNull(fingerprint); + assertEquals(64, fingerprint.length()); // SHA-256 hex string length + assertEquals("b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", fingerprint); + } + + @Test + void getCertificateFingerprint_emptyFile_returnsValidSha256() throws IOException { + Path emptyFile = tempDir.resolve("empty.crt"); + Files.write(emptyFile, new byte[0]); + + String fingerprint = MtlsUtils.getCertificateFingerprint(emptyFile.toString()); + assertEquals("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", fingerprint); + } + + @Test + void getCertificateFingerprint_invalidOrNull_returnsNull() { + assertNull(MtlsUtils.getCertificateFingerprint(null)); + assertNull(MtlsUtils.getCertificateFingerprint("/nonexistent/file.crt")); + assertNull(MtlsUtils.getCertificateFingerprint(tempDir.toString())); // Directory + } + + // --- GKE Credential Bundle Tests --- + + private PropertyProvider homeInTempDir() { + return (name, def) -> { + if ("user.home".equals(name)) return tempDir.resolve("home").toString(); + if ("os.name".equals(name)) return "Linux"; + return def; + }; + } + + private Path writeGkeBundle() throws IOException { + Path bundle = tempDir.resolve("x509.credential-bundle.private-key.pem"); + Files.write(bundle, "bundle".getBytes()); + MtlsUtils.setGkeCredentialBundlePathForTesting(bundle.toString()); + return bundle; + } + + private void writeDefaultGcloudConfig(String content) throws IOException { + Path gcloudDir = tempDir.resolve("home").resolve(".config").resolve("gcloud"); + Files.createDirectories(gcloudDir); + Files.write(gcloudDir.resolve("certificate_config.json"), content.getBytes()); + } + + @Test + void getWorkloadCertPath_noConfig_gkeBundlePresent_returnsBundlePath() throws IOException { + Path bundle = writeGkeBundle(); + EnvironmentProvider envProvider = name -> null; + + assertEquals(bundle.toString(), MtlsUtils.getWorkloadCertPath(envProvider, homeInTempDir())); + assertTrue(MtlsUtils.useMtlsClientCertificate(envProvider, homeInTempDir())); + } + + @Test + void getWorkloadCertPath_noConfig_gkeBundleAbsent_returnsNull() { + MtlsUtils.setGkeCredentialBundlePathForTesting(tempDir.resolve("missing.pem").toString()); + EnvironmentProvider envProvider = name -> null; + + assertNull(MtlsUtils.getWorkloadCertPath(envProvider, homeInTempDir())); + assertFalse(MtlsUtils.useMtlsClientCertificate(envProvider, homeInTempDir())); + } + + @Test + void getWorkloadCertPath_gkeBundleIsDirectory_returnsNull() throws IOException { + Path dir = Files.createDirectory(tempDir.resolve("x509.credential-bundle.private-key.pem")); + MtlsUtils.setGkeCredentialBundlePathForTesting(dir.toString()); + + assertNull(MtlsUtils.getWorkloadCertPath(name -> null, homeInTempDir())); + } + + @Test + void getWorkloadCertPath_gkeBundleEmpty_returnsNull() throws IOException { + Path bundle = Files.createFile(tempDir.resolve("x509.credential-bundle.private-key.pem")); + MtlsUtils.setGkeCredentialBundlePathForTesting(bundle.toString()); + EnvironmentProvider envProvider = name -> null; + + assertNull(MtlsUtils.getWorkloadCertPath(envProvider, homeInTempDir())); + assertFalse(MtlsUtils.useMtlsClientCertificate(envProvider, homeInTempDir())); + assertNull(MtlsUtils.getGkeCredentialBundlePath(envProvider, homeInTempDir())); + } + + @Test + void getWorkloadCertPath_useClientCertFalse_ignoresGkeBundle() throws IOException { + writeGkeBundle(); + EnvironmentProvider envProvider = + name -> "GOOGLE_API_USE_CLIENT_CERTIFICATE".equals(name) ? "false" : null; + + assertNull(MtlsUtils.getWorkloadCertPath(envProvider, homeInTempDir())); + assertFalse(MtlsUtils.useMtlsClientCertificate(envProvider, homeInTempDir())); + } + + @Test + void getWorkloadCertPath_useClientCertNonTrueValues_disablesMtls() throws IOException { + writeGkeBundle(); + for (String disabledVal : new String[] {"0", "no", "FALSE", "off", "random"}) { + EnvironmentProvider envProvider = + name -> "GOOGLE_API_USE_CLIENT_CERTIFICATE".equals(name) ? disabledVal : null; + assertNull(MtlsUtils.getWorkloadCertPath(envProvider, homeInTempDir())); + assertFalse(MtlsUtils.useMtlsClientCertificate(envProvider, homeInTempDir())); + } + } + + @Test + void getWorkloadCertPath_defaultConfigWithoutWorkload_ignoresGkeBundle() throws IOException { + writeGkeBundle(); + writeDefaultGcloudConfig("{\"cert_configs\":{\"enterprise_certificates\":{\"libs\":[]}}}"); + + assertNull(MtlsUtils.getWorkloadCertPath(name -> null, homeInTempDir())); + } + + @Test + void getWorkloadCertPath_explicitConfig_takesPrecedenceOverGkeBundle() throws IOException { + writeGkeBundle(); + Path certFile = Files.write(tempDir.resolve("cert.pem"), "cert".getBytes()); + Path keyFile = Files.write(tempDir.resolve("key.pem"), "key".getBytes()); + Path configFile = tempDir.resolve("config.json"); + Files.write( + configFile, + ("{\"cert_configs\":{\"workload\":{\"cert_path\":\"" + + certFile + + "\",\"key_path\":\"" + + keyFile + + "\"}}}") + .getBytes()); + EnvironmentProvider envProvider = + name -> "GOOGLE_API_CERTIFICATE_CONFIG".equals(name) ? configFile.toString() : null; + + assertEquals(certFile.toString(), MtlsUtils.getWorkloadCertPath(envProvider, homeInTempDir())); + } + + @Test + void hasCertificateConfiguration_reflectsExplicitEnvAndDefaultFile() throws IOException { + assertFalse(MtlsUtils.hasCertificateConfiguration(name -> null, homeInTempDir())); + assertTrue( + MtlsUtils.hasCertificateConfiguration( + name -> "GOOGLE_API_CERTIFICATE_CONFIG".equals(name) ? "/any/path.json" : null, + homeInTempDir())); + + writeDefaultGcloudConfig("{}"); + assertTrue(MtlsUtils.hasCertificateConfiguration(name -> null, homeInTempDir())); + } + + @Test + void getGkeCredentialBundlePath_onlyWhenNoConfiguration() throws IOException { + Path bundle = writeGkeBundle(); + assertEquals( + bundle.toString(), MtlsUtils.getGkeCredentialBundlePath(name -> null, homeInTempDir())); + + writeDefaultGcloudConfig("{}"); + assertNull(MtlsUtils.getGkeCredentialBundlePath(name -> null, homeInTempDir())); + } } diff --git a/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/SecureConnectProviderTest.java b/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/SecureConnectProviderTest.java index fab805fa9520..87e1e23a8336 100644 --- a/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/SecureConnectProviderTest.java +++ b/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/SecureConnectProviderTest.java @@ -35,9 +35,12 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.google.auth.TestUtils; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.security.KeyStore; import java.util.List; import org.junit.jupiter.api.Test; @@ -47,10 +50,16 @@ private static class TestCertProviderCommandProcess extends Process { private final boolean runForever; private final int exitValue; + private final InputStream stdout; public TestCertProviderCommandProcess(int exitValue, boolean runForever) { + this(exitValue, runForever, null); + } + + TestCertProviderCommandProcess(int exitValue, boolean runForever, InputStream stdout) { this.runForever = runForever; this.exitValue = exitValue; + this.stdout = stdout; } @Override @@ -60,7 +69,7 @@ public OutputStream getOutputStream() { @Override public InputStream getInputStream() { - return null; + return stdout; } @Override @@ -101,6 +110,55 @@ public Process createProcess(InputStream metadata) throws IOException { } } + @Test + void testGetKeyStore_certificateChain_keepsFullChain() throws Exception { + String stdout = + MtlsKeyStoreUtilsTest.read(MtlsKeyStoreUtilsTest.CHAIN_CERT_PATH) + + "\n" + + MtlsKeyStoreUtilsTest.read(MtlsKeyStoreUtilsTest.CHAIN_KEY_PATH); + SecureConnectProvider.ProcessProvider processProvider = + metadata -> + new TestCertProviderCommandProcess(0, false, TestUtils.stringToInputStream(stdout)); + + KeyStore store = + SecureConnectProvider.getKeyStore(TestUtils.stringToInputStream("{}"), processProvider); + + MtlsKeyStoreUtilsTest.assertLeafThenIntermediate(store); + } + + @Test + void testGetKeyStore_stdoutHeldOpenAfterBufferedBytes_doesNotReadToEof() throws Exception { + String stdout = + MtlsKeyStoreUtilsTest.read(MtlsKeyStoreUtilsTest.CHAIN_CERT_PATH) + + "\n" + + MtlsKeyStoreUtilsTest.read(MtlsKeyStoreUtilsTest.CHAIN_KEY_PATH); + InputStream neverEndingStdout = + new FilterInputStream(TestUtils.stringToInputStream(stdout)) { + @Override + public int read() throws IOException { + if (available() == 0) { + throw new AssertionError("Attempted to read past buffered stdout bytes"); + } + return super.read(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (available() == 0) { + throw new AssertionError("Attempted to read past buffered stdout bytes"); + } + return super.read(b, off, len); + } + }; + SecureConnectProvider.ProcessProvider processProvider = + metadata -> new TestCertProviderCommandProcess(0, false, neverEndingStdout); + + KeyStore store = + SecureConnectProvider.getKeyStore(TestUtils.stringToInputStream("{}"), processProvider); + + MtlsKeyStoreUtilsTest.assertLeafThenIntermediate(store); + } + @Test void testGetKeyStoreNonZeroExitCode() { try (InputStream metadata = diff --git a/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/X509ProviderTest.java b/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/X509ProviderTest.java index 5ddd1a169d29..e9d7182cf310 100644 --- a/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/X509ProviderTest.java +++ b/google-auth-library-java/oauth2_http/javatests/com/google/auth/mtls/X509ProviderTest.java @@ -31,7 +31,9 @@ package com.google.auth.mtls; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -50,10 +52,12 @@ import java.security.cert.CertificateException; import java.security.cert.CertificateFactory; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; class X509ProviderTest { private static final String TEST_CERT_PATH = "testresources/mtls/test_cert.pem"; + private static final String TEST_KEY_PATH = "testresources/mtls/test_key.pem"; private static final String TEST_CONFIG_PATH = "testresources/mtls/certificate_config.json"; @Test @@ -208,4 +212,231 @@ void x509Provider_malformedCert_throws() throws IOException { assertThrows(Exception.class, testProvider::getKeyStore); } + + @Test + void x509Provider_noConfig_usesGkeCredentialBundle(@TempDir Path tempDir) throws Exception { + Path bundle = writeGkeBundle(tempDir, /* keyFirst= */ false); + MtlsUtils.setGkeCredentialBundlePathForTesting(bundle.toString()); + try { + X509Provider testProvider = + new X509Provider(new TestEnvironmentProvider(), noGcloudConfig(tempDir), null, true); + + KeyStore store = testProvider.getKeyStore(); + assertEquals(1, store.size()); + assertNotNull(store.getCertificateAlias(loadTestCert())); + assertTrue(testProvider.isAvailable()); + } finally { + MtlsUtils.setGkeCredentialBundlePathForTesting(null); + } + } + + @Test + void x509Provider_noConfig_usesGkeCredentialBundleWithKeyFirst(@TempDir Path tempDir) + throws Exception { + Path bundle = writeGkeBundle(tempDir, /* keyFirst= */ true); + MtlsUtils.setGkeCredentialBundlePathForTesting(bundle.toString()); + try { + X509Provider testProvider = + new X509Provider(new TestEnvironmentProvider(), noGcloudConfig(tempDir), null, true); + + KeyStore store = testProvider.getKeyStore(); + assertNotNull(store.getCertificateAlias(loadTestCert())); + } finally { + MtlsUtils.setGkeCredentialBundlePathForTesting(null); + } + } + + @Test + void x509Provider_gkeCredentialBundleWithIntermediate_keepsFullChain(@TempDir Path tempDir) + throws Exception { + Path bundle = + writeGkeBundle( + tempDir, + MtlsKeyStoreUtilsTest.CHAIN_CERT_PATH, + MtlsKeyStoreUtilsTest.CHAIN_KEY_PATH, + /* keyFirst= */ false); + MtlsUtils.setGkeCredentialBundlePathForTesting(bundle.toString()); + try { + X509Provider testProvider = + new X509Provider(new TestEnvironmentProvider(), noGcloudConfig(tempDir), null, true); + + KeyStore store = testProvider.getKeyStore(); + // The intermediate is presented too, so peers that only trust the root can verify the leaf. + MtlsKeyStoreUtilsTest.assertLeafThenIntermediate(store); + } finally { + MtlsUtils.setGkeCredentialBundlePathForTesting(null); + } + } + + @Test + void x509Provider_keyFirstGkeCredentialBundleWithIntermediate_keepsFullChain( + @TempDir Path tempDir) throws Exception { + // GKE pod certificate bundles put the private key before the certificate chain. + Path bundle = + writeGkeBundle( + tempDir, + MtlsKeyStoreUtilsTest.CHAIN_CERT_PATH, + MtlsKeyStoreUtilsTest.CHAIN_KEY_PATH, + /* keyFirst= */ true); + MtlsUtils.setGkeCredentialBundlePathForTesting(bundle.toString()); + try { + X509Provider testProvider = + new X509Provider(new TestEnvironmentProvider(), noGcloudConfig(tempDir), null, true); + + MtlsKeyStoreUtilsTest.assertLeafThenIntermediate(testProvider.getKeyStore()); + } finally { + MtlsUtils.setGkeCredentialBundlePathForTesting(null); + } + } + + @Test + void x509Provider_certFileWithIntermediate_keepsFullChain(@TempDir Path tempDir) + throws Exception { + Path config = tempDir.resolve("certificate_config.json"); + Files.write( + config, + ("{\"cert_configs\":{\"workload\":{\"cert_path\":\"" + + MtlsKeyStoreUtilsTest.CHAIN_CERT_PATH + + "\",\"key_path\":\"" + + MtlsKeyStoreUtilsTest.CHAIN_KEY_PATH + + "\"}}}") + .getBytes(UTF_8)); + + X509Provider testProvider = new X509Provider(config.toString()); + + KeyStore store = testProvider.getKeyStore(); + MtlsKeyStoreUtilsTest.assertLeafThenIntermediate(store); + } + + @Test + void x509Provider_explicitConfigEnv_takesPrecedenceOverGkeCredentialBundle(@TempDir Path tempDir) + throws Exception { + MtlsUtils.setGkeCredentialBundlePathForTesting( + tempDir.resolve("does_not_matter.pem").toString()); + try { + TestEnvironmentProvider envProvider = new TestEnvironmentProvider(); + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", "badfile.txt"); + // Even with a bundle present, an explicit config is used (and fails here). + writeGkeBundleAt(tempDir.resolve("does_not_matter.pem"), false); + X509Provider testProvider = + new X509Provider(envProvider, noGcloudConfig(tempDir), null, true); + + CertificateSourceUnavailableException e = + assertThrows(CertificateSourceUnavailableException.class, testProvider::getKeyStore); + assertFalse(String.valueOf(e.getMessage()).contains("GKE")); + } finally { + MtlsUtils.setGkeCredentialBundlePathForTesting(null); + } + } + + @Test + void x509Provider_defaultGcloudConfig_takesPrecedenceOverGkeCredentialBundle( + @TempDir Path tempDir) throws Exception { + Path bundle = writeGkeBundle(tempDir, false); + MtlsUtils.setGkeCredentialBundlePathForTesting(bundle.toString()); + try { + Path gcloudDir = tempDir.resolve("home").resolve(".config").resolve("gcloud"); + Files.createDirectories(gcloudDir); + Files.write( + gcloudDir.resolve("certificate_config.json"), + "{\"cert_configs\":{\"enterprise_certificates\":{}}}".getBytes(UTF_8)); + + X509Provider testProvider = + new X509Provider(new TestEnvironmentProvider(), noGcloudConfig(tempDir), null, true); + + CertificateSourceUnavailableException e = + assertThrows(CertificateSourceUnavailableException.class, testProvider::getKeyStore); + assertFalse(String.valueOf(e.getMessage()).contains("GKE")); + } finally { + MtlsUtils.setGkeCredentialBundlePathForTesting(null); + } + } + + @Test + void x509Provider_overridePath_ignoresGkeCredentialBundle(@TempDir Path tempDir) + throws Exception { + Path bundle = writeGkeBundle(tempDir, false); + MtlsUtils.setGkeCredentialBundlePathForTesting(bundle.toString()); + try { + X509Provider testProvider = + new X509Provider( + new TestEnvironmentProvider(), noGcloudConfig(tempDir), "badfile.txt", true); + + CertificateSourceUnavailableException e = + assertThrows(CertificateSourceUnavailableException.class, testProvider::getKeyStore); + assertFalse(String.valueOf(e.getMessage()).contains("GKE")); + } finally { + MtlsUtils.setGkeCredentialBundlePathForTesting(null); + } + } + + @Test + void x509Provider_noConfigAndNoGkeCredentialBundle_isUnavailable(@TempDir Path tempDir) + throws Exception { + MtlsUtils.setGkeCredentialBundlePathForTesting(tempDir.resolve("missing.pem").toString()); + try { + X509Provider testProvider = + new X509Provider(new TestEnvironmentProvider(), noGcloudConfig(tempDir), null, true); + + assertThrows(CertificateSourceUnavailableException.class, testProvider::getKeyStore); + assertFalse(testProvider.isAvailable()); + } finally { + MtlsUtils.setGkeCredentialBundlePathForTesting(null); + } + } + + @Test + void x509Provider_publicConstructor_ignoresGkeCredentialBundle(@TempDir Path tempDir) + throws Exception { + Path bundle = writeGkeBundle(tempDir, false); + MtlsUtils.setGkeCredentialBundlePathForTesting(bundle.toString()); + try { + // Non-transport callers (e.g. workload identity federation X.509 credentials) keep the + // existing certificate configuration lookup and never fall back to the GKE bundle. + X509Provider testProvider = + new X509Provider(new TestEnvironmentProvider(), noGcloudConfig(tempDir), null); + + assertThrows(CertificateSourceUnavailableException.class, testProvider::getKeyStore); + assertFalse(testProvider.isAvailable()); + } finally { + MtlsUtils.setGkeCredentialBundlePathForTesting(null); + } + } + + /** Property provider whose user.home has no gcloud certificate_config.json. */ + private static TestPropertyProvider noGcloudConfig(Path tempDir) { + TestPropertyProvider propProvider = new TestPropertyProvider(); + propProvider.setProperty("os.name", "Linux"); + propProvider.setProperty("user.home", tempDir.resolve("home").toString()); + return propProvider; + } + + private static Certificate loadTestCert() throws IOException, CertificateException { + try (FileInputStream fis = new FileInputStream(new File(TEST_CERT_PATH))) { + return CertificateFactory.getInstance("X.509").generateCertificate(fis); + } + } + + private static Path writeGkeBundle(Path dir, boolean keyFirst) throws IOException { + return writeGkeBundle(dir, TEST_CERT_PATH, TEST_KEY_PATH, keyFirst); + } + + private static Path writeGkeBundle(Path dir, String certPath, String keyPath, boolean keyFirst) + throws IOException { + return writeGkeBundleAt( + dir.resolve("x509.credential-bundle.private-key.pem"), certPath, keyPath, keyFirst); + } + + private static Path writeGkeBundleAt(Path bundle, boolean keyFirst) throws IOException { + return writeGkeBundleAt(bundle, TEST_CERT_PATH, TEST_KEY_PATH, keyFirst); + } + + private static Path writeGkeBundleAt( + Path bundle, String certPath, String keyPath, boolean keyFirst) throws IOException { + String cert = new String(Files.readAllBytes(new File(certPath).toPath()), UTF_8); + String key = new String(Files.readAllBytes(new File(keyPath).toPath()), UTF_8); + String content = keyFirst ? key + "\n" + cert : cert + "\n" + key; + Files.write(bundle, content.getBytes(UTF_8)); + return bundle; + } } diff --git a/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/AgentIdentityUtilsTest.java b/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/AgentIdentityUtilsTest.java new file mode 100644 index 000000000000..26680489dd25 --- /dev/null +++ b/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/AgentIdentityUtilsTest.java @@ -0,0 +1,2210 @@ +/* + * Copyright 2026, Google Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.auth.oauth2; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.FileTime; +import java.nio.file.attribute.PosixFilePermissions; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.PrivateKey; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.security.interfaces.ECPublicKey; +import java.security.spec.ECParameterSpec; +import java.security.spec.ECPoint; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class AgentIdentityUtilsTest { + + private static final String VALID_SPIFFE_ORG = + "spiffe://agents.global.org-12345.system.id.goog/path/to/resource"; + private static final String VALID_SPIFFE_PROJ = + "spiffe://agents.global.proj-98765.system.id.goog/another/path"; + private static final String VALID_SPIFFE_NONPROD_ORG = + "spiffe://agents-nonprod.global.org-12345.system.id.goog/path/to/resource"; + private static final String VALID_SPIFFE_NONPROD_PROJ = + "spiffe://agents-nonprod.global.proj-98765.system.id.goog/another/path"; + private static final String INVALID_SPIFFE_DOMAIN = "spiffe://example.com/workload"; + private static final String INVALID_SPIFFE_FORMAT = + "spiffe://agents.global.org-INVALID.system.id.goog/path"; + + private TestEnvironmentProvider envProvider; + private TestPropertyProvider propertyProvider; + @TempDir private Path tempDir; + + @BeforeEach + void setUp() throws IOException { + envProvider = new TestEnvironmentProvider(); + AgentIdentityUtils.setEnvironmentProvider(envProvider); + AgentIdentityUtils.setTimeService(new FakeTimeService()); + // Resolve the well-known gcloud certificate_config.json under a temp home, so a real config on + // the machine running the tests cannot affect GKE credential bundle discovery. + propertyProvider = new TestPropertyProvider(); + propertyProvider.setProperty("os.name", "Linux"); + propertyProvider.setProperty("user.home", tempDir.resolve("home").toString()); + AgentIdentityUtils.setPropertyProvider(propertyProvider); + } + + @AfterEach + void tearDown() throws IOException { + AgentIdentityUtils.resetForTest(); + } + + @Test + public void shouldRequestBoundToken_validOrgSpiffe_returnsTrue() throws CertificateException { + assertTrue( + AgentIdentityCertificateValidationUtils.shouldRequestBoundToken( + mockCertWithSanUri(VALID_SPIFFE_ORG))); + } + + @Test + public void shouldRequestBoundToken_validProjSpiffe_returnsTrue() throws CertificateException { + assertTrue( + AgentIdentityCertificateValidationUtils.shouldRequestBoundToken( + mockCertWithSanUri(VALID_SPIFFE_PROJ))); + } + + @Test + public void shouldRequestBoundToken_validNonprodOrgSpiffe_returnsTrue() + throws CertificateException { + assertTrue( + AgentIdentityCertificateValidationUtils.shouldRequestBoundToken( + mockCertWithSanUri(VALID_SPIFFE_NONPROD_ORG))); + } + + @Test + public void shouldRequestBoundToken_validNonprodProjSpiffe_returnsTrue() + throws CertificateException { + assertTrue( + AgentIdentityCertificateValidationUtils.shouldRequestBoundToken( + mockCertWithSanUri(VALID_SPIFFE_NONPROD_PROJ))); + } + + @Test + public void shouldRequestBoundToken_invalidDomain_returnsFalse() throws CertificateException { + assertFalse( + AgentIdentityCertificateValidationUtils.shouldRequestBoundToken( + mockCertWithSanUri(INVALID_SPIFFE_DOMAIN))); + } + + @Test + public void shouldRequestBoundToken_invalidFormat_returnsFalse() throws CertificateException { + assertFalse( + AgentIdentityCertificateValidationUtils.shouldRequestBoundToken( + mockCertWithSanUri(INVALID_SPIFFE_FORMAT))); + } + + @Test + public void shouldRequestBoundToken_certificateParsingException_returnsFalse() + throws java.security.cert.CertificateParsingException { + X509Certificate mockCert = mock(X509Certificate.class); + when(mockCert.getSubjectAlternativeNames()) + .thenThrow(new java.security.cert.CertificateParsingException()); + assertFalse(AgentIdentityCertificateValidationUtils.shouldRequestBoundToken(mockCert)); + } + + @Test + public void shouldRequestBoundToken_nonUriSan_returnsFalse() + throws java.security.cert.CertificateParsingException { + X509Certificate mockCert = mock(X509Certificate.class); + List dnsSan = Arrays.asList(2, "www.example.com"); + when(mockCert.getSubjectAlternativeNames()).thenReturn(Collections.>singleton(dnsSan)); + assertFalse(AgentIdentityCertificateValidationUtils.shouldRequestBoundToken(mockCert)); + } + + @Test + public void shouldRequestBoundToken_noSan_returnsFalse() throws CertificateException { + X509Certificate mockCert = mock(X509Certificate.class); + when(mockCert.getSubjectAlternativeNames()).thenReturn(null); + assertFalse(AgentIdentityCertificateValidationUtils.shouldRequestBoundToken(mockCert)); + } + + private X509Certificate mockCertWithSanUri(String uri) throws CertificateException { + X509Certificate mockCert = mock(X509Certificate.class); + List spiffeEntry = Arrays.asList(6, uri); + Collection> sans = Collections.singletonList(spiffeEntry); + when(mockCert.getSubjectAlternativeNames()).thenReturn(sans); + return mockCert; + } + + @Test + public void getAgentIdentityCertInfo_enableRuntimeBoundTokenFalse_returnsNullImmediately() + throws IOException { + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "false"); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, "/non/existent/path"); + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void getAgentIdentityCertInfo_legacyPreventSharingFalse_returnsNullImmediately() + throws IOException { + envProvider.setEnv( + AgentIdentityUtils.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, "false"); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, "/non/existent/path"); + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void getAgentIdentityCertInfo_useClientCertFalse_returnsNullImmediatelyWithoutPolling() + throws Exception { + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_USE_CLIENT_CERTIFICATE, "false"); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, "/non/existent/path"); + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_modernVarOverridesLegacyVar() throws Exception { + setupValidAgentCredentialsInTempDir(); + // Modern is true, Legacy is false -> should NOT opt out + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "true"); + envProvider.setEnv( + AgentIdentityUtils.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, "false"); + assertNotNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + + // Modern is false, Legacy is true -> should opt out + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "false"); + envProvider.setEnv( + AgentIdentityUtils.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, "true"); + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void getAgentIdentityCertInfo_enableRuntimeBoundTokenTrue_doesNotOptOut() + throws Exception { + setupValidAgentCredentialsInTempDir(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "true"); + + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info); + assertEquals( + new String(Files.readAllBytes(tempDir.resolve("certificates.pem")), StandardCharsets.UTF_8), + info.getCertContent()); + } + + @Test + public void getAgentIdentityCertInfo_legacyPreventSharingTrue_doesNotOptOut() throws Exception { + setupValidAgentCredentialsInTempDir(); + envProvider.setEnv( + AgentIdentityUtils.GOOGLE_API_PREVENT_AGENT_TOKEN_SHARING_FOR_GCP_SERVICES, "true"); + + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info); + assertEquals( + new String(Files.readAllBytes(tempDir.resolve("certificates.pem")), StandardCharsets.UTF_8), + info.getCertContent()); + } + + @Test + public void getAgentIdentityCertInfo_bothUnset_defaultsToEnabled() throws Exception { + setupValidAgentCredentialsInTempDir(); + + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info); + assertEquals( + new String(Files.readAllBytes(tempDir.resolve("certificates.pem")), StandardCharsets.UTF_8), + info.getCertContent()); + } + + private void setupValidAgentCredentialsInTempDir() throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + + Path targetCertPath = + copyResource("agent/agent_spiffe_cert.pem", tempDir.resolve("certificates.pem")); + Path targetKeyPath = + copyResource("agent/agent_spiffe_key.pem", tempDir.resolve("private_key.pem")); + Path configFile = tempDir.resolve("credentials.json"); + writeWorkloadConfig(configFile, targetCertPath, targetKeyPath); + + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toString()); + } + + @Test + public void getAgentIdentityCertInfo_noConfigEnvVar_returnsNull() throws IOException { + AgentIdentityUtils.setWellKnownDir(tempDir.resolve("non_existent").toString()); + AgentIdentityUtils.setTimeService(new FakeTimeService()); + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void getAgentIdentityCertInfo_happyPath_loadsCertificate() throws Exception { + URL certUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + assertNotNull(certUrl, "Test resource agent/agent_spiffe_cert.pem not found"); + String certPath = Paths.get(certUrl.toURI()).toAbsolutePath().toString(); + + URL keyUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_key.pem"); + assertNotNull(keyUrl, "Test resource agent/agent_spiffe_key.pem not found"); + String keyPath = Paths.get(keyUrl.toURI()).toAbsolutePath().toString(); + + File configFile = tempDir.resolve("config.json").toFile(); + String configJson = + "{" + + " \"cert_configs\": {" + + " \"workload\": {" + + " \"cert_path\": \"" + + certPath.replace("\\", "\\\\") + + "\"," + + " \"key_path\": \"" + + keyPath.replace("\\", "\\\\") + + "\"" + + " }" + + " }" + + "}"; + try (FileOutputStream fos = new FileOutputStream(configFile)) { + fos.write(configJson.getBytes(StandardCharsets.UTF_8)); + } + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", configFile.getAbsolutePath()); + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info); + assertTrue( + AgentIdentityCertificateValidationUtils.shouldRequestBoundToken(info.getCertificate())); + } + + @Test + public void getAgentIdentityCertInfo_timeout_throwsIOException() { + envProvider.setEnv( + "GOOGLE_API_CERTIFICATE_CONFIG", + tempDir.resolve("missing.json").toAbsolutePath().toString()); + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + AgentIdentityUtils.setTimeService(new FakeTimeService()); + IOException e = assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertTrue( + e.getMessage() + .contains( + "Unable to find Agent Identity certificate config or file for bound token request" + + " after multiple retries.")); + } + + @Test + public void getAgentIdentityCertInfo_malformedJson_throwsIOException(@TempDir Path outsideDir) + throws IOException { + File configFile = outsideDir.resolve("config.json").toFile(); + try (FileOutputStream fos = new FileOutputStream(configFile)) { + fos.write("{ \"cert_configs\": invalid json} ".getBytes(StandardCharsets.UTF_8)); + } + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", configFile.getAbsolutePath()); + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + IOException e = assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertTrue(e.getMessage().contains("Failed to parse Agent Identity config JSON")); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_configExists_certMissing_throwsIOExceptionAfterRetries() + throws IOException { + File configFile = tempDir.resolve("config.json").toFile(); + try (FileOutputStream fos = new FileOutputStream(configFile)) { + String json = + "{ \"cert_configs\": { \"workload\": { \"cert_path\": \"/non/existent/cert.pem\"," + + " \"key_path\": \"/non/existent/key.pem\" } } }"; + fos.write(json.getBytes(StandardCharsets.UTF_8)); + } + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", configFile.getAbsolutePath()); + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + AgentIdentityUtils.setTimeService(new FakeTimeService()); + + IOException e = assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertTrue( + e.getMessage() + .contains( + "Unable to find Agent Identity certificate config or file for bound token request" + + " after multiple retries.")); + } + + @Test + public void shouldEnableMtls_true_certsPresent_returnsTrue() throws IOException { + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + assertTrue(AgentIdentityUtils.shouldEnableMtls(true, true)); + } + + @Test + public void shouldEnableMtls_true_certsMissing_throwsIOException() { + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + assertThrows(IOException.class, () -> AgentIdentityUtils.shouldEnableMtls(false, true)); + } + + @Test + public void shouldEnableMtls_false_certsPresent_returnsFalse() throws IOException { + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false"); + assertFalse(AgentIdentityUtils.shouldEnableMtls(true, true)); + } + + @Test + public void shouldEnableMtls_nonTrueValue_certsPresent_returnsFalse() throws IOException { + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "invalid"); + assertFalse(AgentIdentityUtils.shouldEnableMtls(true, true)); + } + + @Test + public void shouldEnableMtls_unset_certsPresent_returnsTrue() throws IOException { + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", null); + assertTrue(AgentIdentityUtils.shouldEnableMtls(true, true)); + } + + @Test + public void shouldEnableMtls_unset_certsPresentNoConfig_returnsFalse() throws IOException { + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", null); + assertFalse(AgentIdentityUtils.shouldEnableMtls(true, false)); + } + + @Test + public void shouldEnableMtls_true_noCertsNoConfig_returnsFalse() throws IOException { + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + assertFalse(AgentIdentityUtils.shouldEnableMtls(false, false)); + } + + @Test + public void shouldEnableMtls_false_noCertsNoConfig_returnsFalse() throws IOException { + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false"); + assertFalse(AgentIdentityUtils.shouldEnableMtls(false, false)); + } + + @Test + public void shouldEnableMtls_unset_noCertsNoConfig_returnsFalse() throws IOException { + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", null); + assertFalse(AgentIdentityUtils.shouldEnableMtls(false, false)); + } + + @Test + public void shouldEnableMtls_unset_certsMissing_configExists_throwsIOException() { + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", null); + assertThrows(IOException.class, () -> AgentIdentityUtils.shouldEnableMtls(false, true)); + } + + @Test + public void getAgentIdentityCertInfo_fallbackPath_loadsCertificate() throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + + URL certUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + assertNotNull(certUrl, "Test resource agent/agent_spiffe_cert.pem not found"); + String certPath = Paths.get(certUrl.toURI()).toAbsolutePath().toString(); + Files.copy(Paths.get(certPath), tempDir.resolve("certificates.pem")); + + URL keyUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_key.pem"); + assertNotNull(keyUrl, "Test resource agent/agent_spiffe_key.pem not found"); + String keyPath = Paths.get(keyUrl.toURI()).toAbsolutePath().toString(); + Files.copy(Paths.get(keyPath), tempDir.resolve("private_key.pem")); + + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", null); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", null); + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info); + assertEquals( + new String(Files.readAllBytes(tempDir.resolve("certificates.pem")), StandardCharsets.UTF_8), + info.getCertContent()); + } + + @Test + public void verifyKeyPair_match_returnsTrue() throws Exception { + KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); + kpg.initialize(2048); + KeyPair kp = kpg.generateKeyPair(); + + X509Certificate mockCert = mock(X509Certificate.class); + when(mockCert.getPublicKey()).thenReturn(kp.getPublic()); + + assertTrue(AgentIdentityCertificateValidationUtils.verifyKeyPair(mockCert, kp.getPrivate())); + } + + @Test + public void verifyKeyPair_ecKey_returnsTrue() throws Exception { + KeyPairGenerator kpg = KeyPairGenerator.getInstance("EC"); + kpg.initialize(256); + KeyPair kp = kpg.generateKeyPair(); + + X509Certificate ecCert = mock(X509Certificate.class); + when(ecCert.getPublicKey()).thenReturn(kp.getPublic()); + assertTrue(AgentIdentityCertificateValidationUtils.verifyKeyPair(ecCert, kp.getPrivate())); + } + + @Test + public void shouldRequestBoundToken_uppercaseSpiffeSchemeAndTrustDomain_returnsTrue() + throws Exception { + X509Certificate mockCert = mock(X509Certificate.class); + Collection> sans = + Collections.singletonList( + Arrays.asList(6, "SPIFFE://AGENTS.GLOBAL.ORG-12345.SYSTEM.ID.GOOG/ns/default/sa/test")); + when(mockCert.getSubjectAlternativeNames()).thenReturn(sans); + + assertTrue(AgentIdentityCertificateValidationUtils.shouldRequestBoundToken(mockCert)); + } + + @Test + public void verifyKeyPair_mismatch_returnsFalse() throws Exception { + KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); + kpg.initialize(2048); + KeyPair kp1 = kpg.generateKeyPair(); + KeyPair kp2 = kpg.generateKeyPair(); + + X509Certificate mockCert = mock(X509Certificate.class); + when(mockCert.getPublicKey()).thenReturn(kp1.getPublic()); + + assertFalse(AgentIdentityCertificateValidationUtils.verifyKeyPair(mockCert, kp2.getPrivate())); + } + + @Test + public void getAgentIdentityCertInfo_mismatch_throwsIOExceptionAfterRetries() throws Exception { + URL certUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + assertNotNull(certUrl, "Test resource agent/agent_spiffe_cert.pem not found"); + String certPath = Paths.get(certUrl.toURI()).toAbsolutePath().toString(); + + // Generate a random key that won't match the cert + KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); + kpg.initialize(2048); + KeyPair kp = kpg.generateKeyPair(); + + File keyFile = tempDir.resolve("private_key.pem").toFile(); + String keyPem = + "-----BEGIN PRIVATE KEY-----\n" + + java.util.Base64.getMimeEncoder().encodeToString(kp.getPrivate().getEncoded()) + + "\n-----END PRIVATE KEY-----"; + try (FileOutputStream fos = new FileOutputStream(keyFile)) { + fos.write(keyPem.getBytes(StandardCharsets.UTF_8)); + } + + File configFile = tempDir.resolve("config.json").toFile(); + String configJson = + "{" + + " \"cert_configs\": {" + + " \"workload\": {" + + " \"cert_path\": \"" + + certPath.replace("\\", "\\\\") + + "\"," + + " \"key_path\": \"" + + keyFile.getAbsolutePath().replace("\\", "\\\\") + + "\"" + + " }" + + " }" + + "}"; + try (FileOutputStream fos = new FileOutputStream(configFile)) { + fos.write(configJson.getBytes(StandardCharsets.UTF_8)); + } + + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", configFile.getAbsolutePath()); + + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + GoogleAuthException e = + assertThrows(GoogleAuthException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertTrue(e.isRetryable()); + assertEquals(3, e.getRetryCount()); + assertTrue( + e.getMessage() + .contains( + "Agent Identity certificate and private key mismatch or read failure after 3" + + " attempts.")); + assertEquals(200, fakeTime.currentTimeMillis()); // 2 retries * 100ms + } + + @Test + public void + getAgentIdentityCertInfo_wellKnownDirExistsNoFiles_notExplicitlyEnabled_returnsNullImmediately() + throws IOException { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", null); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", null); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void + getAgentIdentityCertInfo_wellKnownDirExistsNoFiles_explicitlyEnabled_returnsNullWithoutPolling() + throws IOException { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", null); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void + getAgentIdentityCertInfo_wellKnownDirMissing_explicitlyEnabled_returnsNullWithoutPolling() + throws IOException { + AgentIdentityUtils.setWellKnownDir( + tempDir.resolve("non_existent_well_known").toAbsolutePath().toString() + "/"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", null); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void readCertificateChain_combinedBundle_stripsPrivateKeyAndRetainsCertificates() + throws Exception { + URL cert1Url = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + assertNotNull(cert1Url); + String cert1Pem = + new String(Files.readAllBytes(Paths.get(cert1Url.toURI())), StandardCharsets.UTF_8); + + URL cert2Url = getClass().getClassLoader().getResource("x509_leaf_certificate.pem"); + assertNotNull(cert2Url); + String cert2Pem = + new String(Files.readAllBytes(Paths.get(cert2Url.toURI())), StandardCharsets.UTF_8); + + URL keyUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_key.pem"); + assertNotNull(keyUrl); + String keyPem = + new String(Files.readAllBytes(Paths.get(keyUrl.toURI())), StandardCharsets.UTF_8); + + // Create a bundle file with cert1 then private key then cert2 + String bundleContent = cert1Pem + "\n" + keyPem + "\n" + cert2Pem; + Path bundleFile = tempDir.resolve("bundle.pem"); + Files.write(bundleFile, bundleContent.getBytes(StandardCharsets.UTF_8)); + + String extractedChain = + AgentIdentityCertificateValidationUtils.readCertificateChain(bundleFile.toString()); + + // Verify both certificates are retained + assertTrue(extractedChain.contains(cert1Pem.trim())); + assertTrue(extractedChain.contains(cert2Pem.trim())); + // Verify private key is stripped + assertFalse(extractedChain.contains("PRIVATE KEY")); + assertFalse(extractedChain.contains("BEGIN PRIVATE KEY")); + assertFalse(extractedChain.contains("END PRIVATE KEY")); + } + + @Test + public void readCertificateChain_noCertificates_throwsIOException() throws Exception { + URL keyUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_key.pem"); + assertNotNull(keyUrl); + String keyPem = + new String(Files.readAllBytes(Paths.get(keyUrl.toURI())), StandardCharsets.UTF_8); + + Path keyOnlyFile = tempDir.resolve("key_only.pem"); + Files.write(keyOnlyFile, keyPem.getBytes(StandardCharsets.UTF_8)); + + IOException e = + assertThrows( + IOException.class, + () -> + AgentIdentityCertificateValidationUtils.readCertificateChain( + keyOnlyFile.toString())); + assertTrue(e.getMessage().contains("No PEM certificates found in certificate file")); + } + + @Test + public void loadAndVerifyCredentials_validCombinedBundle_verifiesKeyAndStripsPrivateKey() + throws Exception { + URL certUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + assertNotNull(certUrl); + String certPem = + new String(Files.readAllBytes(Paths.get(certUrl.toURI())), StandardCharsets.UTF_8); + + URL keyUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_key.pem"); + assertNotNull(keyUrl); + String keyPem = + new String(Files.readAllBytes(Paths.get(keyUrl.toURI())), StandardCharsets.UTF_8); + + String bundleContent = certPem + "\n" + keyPem; + Path bundleFile = tempDir.resolve("credentialbundle.pem"); + Files.write(bundleFile, bundleContent.getBytes(StandardCharsets.UTF_8)); + + AgentIdentityUtils.CertInfo info = + AgentIdentityUtils.loadAndVerifyCredentials(bundleFile.toString(), bundleFile.toString()); + assertNotNull(info); + assertNotNull(info.getCertificate()); + assertTrue( + AgentIdentityCertificateValidationUtils.shouldRequestBoundToken(info.getCertificate())); + assertTrue(info.getCertContent().contains("BEGIN CERTIFICATE")); + assertFalse(info.getCertContent().contains("PRIVATE KEY")); + } + + @Test + public void loadAndVerifyCredentials_bundleWithMismatchedKey_failsVerification() + throws Exception { + URL certUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + assertNotNull(certUrl); + String certPem = + new String(Files.readAllBytes(Paths.get(certUrl.toURI())), StandardCharsets.UTF_8); + + // Generate a random key that does not match the cert + KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); + kpg.initialize(2048); + KeyPair kp = kpg.generateKeyPair(); + String mismatchedKeyPem = + "-----BEGIN PRIVATE KEY-----\n" + + java.util.Base64.getMimeEncoder().encodeToString(kp.getPrivate().getEncoded()) + + "\n-----END PRIVATE KEY-----"; + + String bundleContent = certPem + "\n" + mismatchedKeyPem; + Path bundleFile = tempDir.resolve("credentialbundle.pem"); + Files.write(bundleFile, bundleContent.getBytes(StandardCharsets.UTF_8)); + + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + AgentIdentityUtils.setTimeService(new FakeTimeService()); + + GoogleAuthException e = + assertThrows( + GoogleAuthException.class, + () -> + AgentIdentityUtils.loadAndVerifyCredentials( + bundleFile.toString(), bundleFile.toString())); + assertTrue(e.isRetryable()); + assertEquals(3, e.getRetryCount()); + assertTrue( + e.getMessage() + .contains( + "Agent Identity certificate and private key mismatch or read failure after 3" + + " attempts.")); + } + + @Test + public void getAgentIdentityCertInfo_implicitDiscovery_bundleWithMismatchedKey_throwsIOException() + throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + URL certUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + assertNotNull(certUrl); + String certPem = + new String(Files.readAllBytes(Paths.get(certUrl.toURI())), StandardCharsets.UTF_8); + + KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); + kpg.initialize(2048); + KeyPair kp = kpg.generateKeyPair(); + String mismatchedKeyPem = + "-----BEGIN PRIVATE KEY-----\n" + + java.util.Base64.getMimeEncoder().encodeToString(kp.getPrivate().getEncoded()) + + "\n-----END PRIVATE KEY-----"; + + String bundleContent = certPem + "\n" + mismatchedKeyPem; + Path bundleFile = tempDir.resolve("credentialbundle.pem"); + Files.write(bundleFile, bundleContent.getBytes(StandardCharsets.UTF_8)); + + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", null); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + AgentIdentityUtils.setTimeService(new FakeTimeService()); + + GoogleAuthException e = + assertThrows(GoogleAuthException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertTrue(e.isRetryable()); + assertEquals(3, e.getRetryCount()); + assertNotNull(e.getCause()); + assertTrue(e.getCause().getMessage().contains("Certificate and private key do not match")); + } + + @Test + public void getAgentIdentityCertInfo_wellKnownBundleFile_verifiesAndStripsPrivateKey() + throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + + URL certUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + assertNotNull(certUrl); + String certPem = + new String(Files.readAllBytes(Paths.get(certUrl.toURI())), StandardCharsets.UTF_8); + + URL keyUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_key.pem"); + assertNotNull(keyUrl); + String keyPem = + new String(Files.readAllBytes(Paths.get(keyUrl.toURI())), StandardCharsets.UTF_8); + + String bundleContent = certPem + "\n" + keyPem; + Path bundleFile = tempDir.resolve("credentialbundle.pem"); + Files.write(bundleFile, bundleContent.getBytes(StandardCharsets.UTF_8)); + + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, null); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_USE_CLIENT_CERTIFICATE, "true"); + + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info); + assertNotNull(info.getCertificate()); + assertTrue( + AgentIdentityCertificateValidationUtils.shouldRequestBoundToken(info.getCertificate())); + assertTrue(info.getCertContent().contains("BEGIN CERTIFICATE")); + assertFalse(info.getCertContent().contains("PRIVATE KEY")); + } + + @Test + public void getAgentIdentityCertInfo_explicitConfigDirectoryAccessDenied_throwsIOException() + throws Exception { + Path restrictedDir = tempDir.resolve("restricted_dir"); + Files.createDirectory(restrictedDir); + Path configFile = restrictedDir.resolve("config.json"); + Files.write(configFile, "{}".getBytes(StandardCharsets.UTF_8)); + + restrictedDir.toFile().setReadable(false, false); + restrictedDir.toFile().setExecutable(false, false); + + try { + Assumptions.assumeFalse(Files.isReadable(configFile)); + envProvider.setEnv( + AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toAbsolutePath().toString()); + assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + } finally { + restrictedDir.toFile().setReadable(true, false); + restrictedDir.toFile().setExecutable(true, false); + } + } + + @Test + public void getAgentIdentityCertInfo_explicitConfigFileNotReadable_throwsIOException() + throws Exception { + Path configFile = tempDir.resolve("unreadable_config.json"); + Files.write(configFile, "{}".getBytes(StandardCharsets.UTF_8)); + configFile.toFile().setReadable(false, false); + + try { + Assumptions.assumeFalse(Files.isReadable(configFile)); + envProvider.setEnv( + AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toAbsolutePath().toString()); + assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + } finally { + configFile.toFile().setReadable(true, false); + } + } + + @Test + public void getAgentIdentityCertInfo_implicitWellKnownAccessDenied_returnsNull() + throws Exception { + Path restrictedDir = tempDir.resolve("restricted_well_known"); + Files.createDirectory(restrictedDir); + Path bundleFile = restrictedDir.resolve("credentialbundle.pem"); + Files.write(bundleFile, "test".getBytes(StandardCharsets.UTF_8)); + + restrictedDir.toFile().setReadable(false, false); + restrictedDir.toFile().setExecutable(false, false); + + try { + Assumptions.assumeFalse(Files.isReadable(bundleFile)); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, null); + AgentIdentityUtils.setWellKnownDir(restrictedDir.toAbsolutePath().toString() + "/"); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + } finally { + restrictedDir.toFile().setReadable(true, false); + restrictedDir.toFile().setExecutable(true, false); + } + } + + @Test + public void getAgentIdentityCertInfo_enterpriseConfig_returnsNullImmediatelyWithoutPolling() + throws Exception { + File configFile = tempDir.resolve("enterprise_config.json").toFile(); + try (FileOutputStream fos = new FileOutputStream(configFile)) { + String json = + "{\n" + + " \"cert_configs\": {\n" + + " \"enterprise\": {\n" + + " \"cert_provider_command\": [\"/bin/echo\", \"dummy\"]\n" + + " }\n" + + " }\n" + + "}"; + fos.write(json.getBytes(StandardCharsets.UTF_8)); + } + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", configFile.getAbsolutePath()); + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNull(info); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_configWithoutWorkload_returnsNullImmediatelyWithoutPolling() + throws Exception { + File configFile = tempDir.resolve("empty_cert_configs.json").toFile(); + try (FileOutputStream fos = new FileOutputStream(configFile)) { + String json = "{ \"cert_configs\": {} }"; + fos.write(json.getBytes(StandardCharsets.UTF_8)); + } + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", configFile.getAbsolutePath()); + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNull(info); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void + getAgentIdentityCertInfo_missingConfigOutsideWellKnownDir_throwsIOExceptionImmediatelyWithoutPolling( + @TempDir Path outsideDir) throws Exception { + Path missingConfigFile = outsideDir.resolve("missing_config.json"); + envProvider.setEnv( + "GOOGLE_API_CERTIFICATE_CONFIG", missingConfigFile.toAbsolutePath().toString()); + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void + getAgentIdentityCertInfo_configOutsideWellKnownDir_targetCertInWellKnownDir_pollsUntilReady( + @TempDir Path outsideDir) throws Exception { + Path targetCertPath = tempDir.resolve("certificates.pem"); + Path targetKeyPath = tempDir.resolve("private_key.pem"); + + File configFile = outsideDir.resolve("certificate_config.json").toFile(); + String configJson = + "{" + + " \"cert_configs\": {" + + " \"workload\": {" + + " \"cert_path\": \"" + + targetCertPath.toAbsolutePath().toString().replace("\\", "\\\\") + + "\"," + + " \"key_path\": \"" + + targetKeyPath.toAbsolutePath().toString().replace("\\", "\\\\") + + "\"" + + " }" + + " }" + + "}"; + Files.write(configFile.toPath(), configJson.getBytes(StandardCharsets.UTF_8)); + + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", configFile.getAbsolutePath()); + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + + FakeTimeService fakeTime = new FakeTimeService(); + fakeTime.setOnSleepCallback( + () -> { + if (fakeTime.getSleepCount() == 2) { + try { + URL certUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + URL keyUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_key.pem"); + Files.copy(Paths.get(certUrl.toURI()), targetCertPath); + Files.copy(Paths.get(keyUrl.toURI()), targetKeyPath); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + AgentIdentityUtils.setTimeService(fakeTime); + + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info); + assertEquals(2, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_partialJsonDuringStartupPoll_retriesAndSucceeds() + throws Exception { + Path targetCertPath = tempDir.resolve("certificates.pem"); + Path targetKeyPath = tempDir.resolve("private_key.pem"); + URL certUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + URL keyUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_key.pem"); + Files.copy(Paths.get(certUrl.toURI()), targetCertPath); + Files.copy(Paths.get(keyUrl.toURI()), targetKeyPath); + + Path configFile = tempDir.resolve("config.json"); + // Write partial/incomplete JSON simulating non-atomic write at startup + Files.write(configFile, "{ \"cert_configs\": ".getBytes(StandardCharsets.UTF_8)); + + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", configFile.toAbsolutePath().toString()); + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + + String validJson = + "{" + + " \"cert_configs\": {" + + " \"workload\": {" + + " \"cert_path\": \"" + + targetCertPath.toAbsolutePath().toString().replace("\\", "\\\\") + + "\"," + + " \"key_path\": \"" + + targetKeyPath.toAbsolutePath().toString().replace("\\", "\\\\") + + "\"" + + " }" + + " }" + + "}"; + + FakeTimeService fakeTime = new FakeTimeService(); + fakeTime.setOnSleepCallback( + () -> { + if (fakeTime.getSleepCount() == 1) { + try { + Files.write(configFile, validJson.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + AgentIdentityUtils.setTimeService(fakeTime); + + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info); + assertEquals(1, fakeTime.getSleepCount()); + } + + @Test + public void + getAgentIdentityCertInfo_configInWellKnownDirTimeout_subsequentRefreshDoesNotPollAndThrowsIOException() + throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + envProvider.setEnv( + "GOOGLE_API_CERTIFICATE_CONFIG", + tempDir.resolve("missing_config.json").toAbsolutePath().toString()); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", null); + + // Initial call polls for all 100 cycles (99 sleeps) and fails closed with IOException + assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertEquals(99, fakeTime.getSleepCount()); + + // Subsequent token refresh fails closed immediately without sleeping + assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertEquals(99, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_zeroByteFilesDuringStartupPoll_waitsUntilNonEmpty() + throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + Path targetCertPath = tempDir.resolve("certificates.pem"); + Path targetKeyPath = tempDir.resolve("private_key.pem"); + // Create 0-byte files simulating CSI driver creating file entries before writing content + Files.createFile(targetCertPath); + Files.createFile(targetKeyPath); + + Path configFile = tempDir.resolve("credentials.json"); + writeWorkloadConfig(configFile, targetCertPath, targetKeyPath); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toString()); + + // Populate after sleep 4 and 5 so loadAndVerifyCredentials's 3 attempts (2 sleeps) cannot + // mask a broken 0-byte startup poll check. + FakeTimeService fakeTime = new FakeTimeService(); + fakeTime.setOnSleepCallback( + () -> { + if (fakeTime.getSleepCount() == 4) { + try { + // First populate only cert; key is still 0 bytes + copyResource("agent/agent_spiffe_cert.pem", targetCertPath); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else if (fakeTime.getSleepCount() == 5) { + try { + // Then populate key + copyResource("agent/agent_spiffe_key.pem", targetKeyPath); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + AgentIdentityUtils.setTimeService(fakeTime); + + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info); + assertEquals(5, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_symlinkedWellKnownDir_pollsForStartupFiles() + throws Exception { + Path realWellKnown = Files.createDirectory(tempDir.resolve("real_run_secrets")); + Path symlinkWellKnown = tempDir.resolve("var_run_secrets"); + try { + Files.createSymbolicLink(symlinkWellKnown, realWellKnown); + } catch (UnsupportedOperationException | IOException e) { + Assumptions.assumeTrue(false, "Symbolic links not supported: " + e.getMessage()); + } + + // Configure wellKnownDir via symlink path (/var/run/...) while GOOGLE_API_CERTIFICATE_CONFIG + // uses the real path (/run/...) for a file that does not exist yet at startup. + AgentIdentityUtils.setWellKnownDir(symlinkWellKnown.toAbsolutePath().toString() + "/"); + Path configViaRealPath = realWellKnown.resolve("credentials.json"); + Path certViaRealPath = realWellKnown.resolve("certificates.pem"); + Path keyViaRealPath = realWellKnown.resolve("private_key.pem"); + envProvider.setEnv( + AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, + configViaRealPath.toAbsolutePath().toString()); + + FakeTimeService fakeTime = new FakeTimeService(); + fakeTime.setOnSleepCallback( + () -> { + if (fakeTime.getSleepCount() == 1) { + try { + copyResource("agent/agent_spiffe_cert.pem", certViaRealPath); + copyResource("agent/agent_spiffe_key.pem", keyViaRealPath); + writeWorkloadConfig(configViaRealPath, certViaRealPath, keyViaRealPath); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + AgentIdentityUtils.setTimeService(fakeTime); + + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info); + assertEquals(1, fakeTime.getSleepCount()); + } + + @Test + public void + getAgentIdentityCertInfo_startupSucceeded_subsequentRefreshMissingFileRetriesAndFallsBackToCache() + throws Exception { + setupValidAgentCredentialsInTempDir(); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", null); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + + // Initial startup discovery succeeds immediately without polling + AgentIdentityUtils.CertInfo info1 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info1); + assertEquals(0, fakeTime.getSleepCount()); + + // Simulate certificate file disappearing on disk after initial startup + Files.delete(tempDir.resolve("certificates.pem")); + + // Subsequent token refresh retries briefly (2 sleeps of 100ms) and falls back to cached cert + AgentIdentityUtils.CertInfo info2 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertSame(info1, info2); + assertEquals(2, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_startupSucceeded_rotationMomentaryUnlinkRecoversDuringRetry() + throws Exception { + setupValidAgentCredentialsInTempDir(); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", null); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + + AgentIdentityUtils.CertInfo info1 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info1); + + Path certFile = tempDir.resolve("certificates.pem"); + byte[] certBytes = Files.readAllBytes(certFile); + // Momentarily unlink certificate file simulating non-atomic unlink+create rotation + Files.delete(certFile); + + fakeTime.setOnSleepCallback( + () -> { + if (fakeTime.getSleepCount() == 1) { + try { + // Re-create the file with updated content during retry sleep + Files.write( + certFile, + (new String(certBytes, StandardCharsets.UTF_8) + "\n") + .getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + + AgentIdentityUtils.CertInfo info2 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info2); + assertNotSame(info1, info2); + assertEquals(1, fakeTime.getSleepCount()); + } + + @Test + public void + getAgentIdentityCertInfo_implicitDiscovery_certificatesPemWithoutPrivateKey_returnsNull() + throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + URL certUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + assertNotNull(certUrl); + Files.copy(Paths.get(certUrl.toURI()), tempDir.resolve("certificates.pem")); + // Without private_key.pem, implicit well-known discovery does not consider certOnlyPath ready + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", null); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", null); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_implicitDiscovery_bundleWithoutPrivateKey_throwsIOException() + throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + URL certUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + assertNotNull(certUrl); + // credentialbundle.pem exists without a valid private key, so discovery fails closed + Files.copy(Paths.get(certUrl.toURI()), tempDir.resolve("credentialbundle.pem")); + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", null); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertEquals(2, fakeTime.getSleepCount()); + } + + @Test + public void + getAgentIdentityCertInfo_explicitMtlsEnabled_missingPrivateKey_returnsNullWithoutPolling() + throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + URL certUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + assertNotNull(certUrl); + Files.copy(Paths.get(certUrl.toURI()), tempDir.resolve("certificates.pem")); + // Intentionally do NOT copy private_key.pem + + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", null); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_certificateOnlyConfig_omittedKeyPath_throwsIOException() + throws Exception { + URL certUrl = getClass().getClassLoader().getResource("agent/agent_spiffe_cert.pem"); + assertNotNull(certUrl); + String certPath = Paths.get(certUrl.toURI()).toAbsolutePath().toString(); + + File configFile = tempDir.resolve("config.json").toFile(); + String configJson = + "{" + + " \"cert_configs\": {" + + " \"workload\": {" + + " \"cert_path\": \"" + + certPath.replace("\\", "\\\\") + + "\"" + + " }" + + " }" + + "}"; + Files.write(configFile.toPath(), configJson.getBytes(StandardCharsets.UTF_8)); + + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", configFile.getAbsolutePath()); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + IOException e = assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertTrue( + e.getMessage() + .contains( + "Unable to find Agent Identity certificate config or file for bound token" + + " request")); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_nonAgentSpiffeCert_returnsNullWithoutReadingKey() + throws Exception { + URL certUrl = getClass().getClassLoader().getResource("x509_leaf_certificate.pem"); + assertNotNull(certUrl); + String certPath = Paths.get(certUrl.toURI()).toAbsolutePath().toString(); + + Path invalidKeyFile = tempDir.resolve("invalid_key.pem"); + Files.write(invalidKeyFile, "not-a-valid-private-key".getBytes(StandardCharsets.UTF_8)); + + File configFile = tempDir.resolve("config.json").toFile(); + String configJson = + "{" + + " \"cert_configs\": {" + + " \"workload\": {" + + " \"cert_path\": \"" + + certPath.replace("\\", "\\\\") + + "\"," + + " \"key_path\": \"" + + invalidKeyFile.toAbsolutePath().toString().replace("\\", "\\\\") + + "\"" + + " }" + + " }" + + "}"; + Files.write(configFile.toPath(), configJson.getBytes(StandardCharsets.UTF_8)); + + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", configFile.getAbsolutePath()); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + // Should return null without attempting to parse the invalid private key or sleeping + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, fakeTime.getSleepCount()); + assertNull(AgentIdentityUtils.getBoundTokenPayload()); + } + + @Test + public void getAgentIdentityCertInfo_cachesVerifiedCertInfoAndInvalidatesOnRotation() + throws Exception { + setupValidAgentCredentialsInTempDir(); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + AgentIdentityUtils.CertInfo info1 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info1); + + // Second call with unchanged files on disk must return the exact same cached instance + AgentIdentityUtils.CertInfo info2 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertSame(info1, info2); + + // Simulate certificate rotation on disk by modifying the file content/size + Path certFile = tempDir.resolve("certificates.pem"); + String originalPem = new String(Files.readAllBytes(certFile), StandardCharsets.UTF_8); + Files.write(certFile, (originalPem + "\n").getBytes(StandardCharsets.UTF_8)); + + AgentIdentityUtils.CertInfo info3 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info3); + assertNotSame(info1, info3); + } + + @Test + public void + getAgentIdentityCertInfo_rotationWithIdenticalByteLengthAndUpdatedMtime_reloadsCredentials() + throws Exception { + setupValidAgentCredentialsInTempDir(); + Path certFile = tempDir.resolve("certificates.pem"); + String basePem = new String(Files.readAllBytes(certFile), StandardCharsets.UTF_8); + + // Write initial version with a fixed-length trailing comment + String v1Pem = basePem + "\n# version-1"; + Files.write(certFile, v1Pem.getBytes(StandardCharsets.UTF_8)); + FileTime initialMtime = Files.getLastModifiedTime(certFile); + long initialSize = Files.size(certFile); + + AgentIdentityUtils.CertInfo info1 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info1); + + // Overwrite with identical byte length but updated mtime + String v2Pem = basePem + "\n# version-2"; + Files.write(certFile, v2Pem.getBytes(StandardCharsets.UTF_8)); + Files.setLastModifiedTime(certFile, FileTime.fromMillis(initialMtime.toMillis() + 5000)); + assertEquals(initialSize, Files.size(certFile)); + + AgentIdentityUtils.CertInfo info2 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info2); + assertNotSame(info1, info2); + } + + @Test + public void + getAgentIdentityCertInfo_concurrentCacheInvalidationDuringRotation_fallsBackToInitialSnapshot() + throws Exception { + setupValidAgentCredentialsInTempDir(); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", null); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + + AgentIdentityUtils.CertInfo info1 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info1); + + // Simulate transient file disappearance during steady-state rotation + Files.delete(tempDir.resolve("certificates.pem")); + + // While the thread sleeps in loadAndVerifyCredentials backoff, simulate another thread + // concurrently clearing/invalidating cachedCredentials + fakeTime.setOnSleepCallback(() -> AgentIdentityCacheUtils.clearCachedAgentIdentityInfo()); + + AgentIdentityUtils.CertInfo info2 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertSame(info1, info2); + } + + @Test + public void + getAgentIdentityCertInfo_concurrentRotationUpdatedByAnotherThread_returnsFreshestCredential() + throws Exception { + setupValidAgentCredentialsInTempDir(); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", null); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + + Path certFile = tempDir.resolve("certificates.pem"); + Path keyFile = tempDir.resolve("private_key.pem"); + String basePem = new String(Files.readAllBytes(certFile), StandardCharsets.UTF_8); + + AgentIdentityUtils.CertInfo info1 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info1); + + // Delete certFile so Thread A enters the retry sleep loop + Files.delete(certFile); + + AtomicReference concurrentRotatedInfo = new AtomicReference<>(); + fakeTime.setOnSleepCallback( + () -> { + if (concurrentRotatedInfo.get() == null) { + try { + // Simulate Thread B completing rotation and updating cachedCredentials while Thread A + // sleeps + String v2Pem = basePem + "\n# rotated-by-thread-B"; + Files.write(certFile, v2Pem.getBytes(StandardCharsets.UTF_8)); + AgentIdentityUtils.CertInfo rotated = + AgentIdentityUtils.loadAndVerifyCredentials( + certFile.toString(), keyFile.toString()); + concurrentRotatedInfo.set(rotated); + // Delete certFile again so Thread A's own disk retries fail and must use cache + // fallback + Files.delete(certFile); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + + AgentIdentityUtils.CertInfo result = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(concurrentRotatedInfo.get()); + assertNotSame(info1, concurrentRotatedInfo.get()); + assertSame(concurrentRotatedInfo.get(), result); + } + + @Test + public void getAgentIdentityCertInfo_concurrentThreadsReadingAndRotating_noNpeOrRaceConditions() + throws Exception { + setupValidAgentCredentialsInTempDir(); + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", null); + envProvider.setEnv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + + Path certFile = tempDir.resolve("certificates.pem"); + Path keyFile = tempDir.resolve("private_key.pem"); + byte[] agentCertBytes = Files.readAllBytes(certFile); + + URL nonAgentUrl = getClass().getClassLoader().getResource("mtlsCertAndKey.pem"); + assertNotNull(nonAgentUrl); + byte[] nonAgentBytes = Files.readAllBytes(Paths.get(nonAgentUrl.toURI())); + + // Prime initial cache + assertNotNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + + int numThreads = 8; + int iterations = 100; + ExecutorService executor = Executors.newFixedThreadPool(numThreads + 1); + try { + List> tasks = new ArrayList<>(); + // Mutator thread rapidly alternates between agent cert, non-agent cert (keyMetadata == null), + // and cache resets + tasks.add( + () -> { + for (int i = 0; i < iterations; i++) { + if (i % 3 == 0) { + Files.write(certFile, nonAgentBytes); + AgentIdentityUtils.loadAndVerifyCredentials( + certFile.toString(), keyFile.toString()); + } else if (i % 3 == 1) { + Files.write(certFile, agentCertBytes); + AgentIdentityUtils.loadAndVerifyCredentials( + certFile.toString(), keyFile.toString()); + } else { + AgentIdentityCacheUtils.clearCachedAgentIdentityInfo(); + } + } + Files.write(certFile, agentCertBytes); + return null; + }); + + // Reader threads concurrently call getAgentIdentityCertInfo() + for (int t = 0; t < numThreads; t++) { + tasks.add( + () -> { + for (int i = 0; i < iterations; i++) { + try { + AgentIdentityUtils.getAgentIdentityCertInfo(); + } catch (IOException ignored) { + // Expected if mutator wrote non-matching cert/key mid-read, but NPE must NEVER + // occur + } + } + return null; + }); + } + + List> futures = executor.invokeAll(tasks); + for (Future future : futures) { + future.get(); // Re-throws any NullPointerException or RuntimeException + } + } finally { + executor.shutdownNow(); + } + } + + @Test + public void getAgentIdentityCertInfo_steadyStateKeyMismatch_fallsBackToCache() throws Exception { + setupValidAgentCredentialsInTempDir(); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + // 1. Initial call succeeds and populates cache + AgentIdentityUtils.CertInfo initialInfo = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(initialInfo); + + // 2. Overwrite private_key.pem on disk with a mismatched key (and advance mtime so cache + // invalidates) + KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); + kpg.initialize(2048); + PrivateKey mismatchedKey = kpg.generateKeyPair().getPrivate(); + String mismatchedKeyPem = + "-----BEGIN PRIVATE KEY-----\n" + + java.util.Base64.getEncoder().encodeToString(mismatchedKey.getEncoded()) + + "\n-----END PRIVATE KEY-----\n"; + Path keyPath = tempDir.resolve("private_key.pem"); + Files.write(keyPath, mismatchedKeyPem.getBytes(StandardCharsets.UTF_8)); + Files.setLastModifiedTime( + keyPath, java.nio.file.attribute.FileTime.fromMillis(System.currentTimeMillis() + 10000)); + + // 3. With steady-state cache populated, lastException is set on verifyKeyPair failure so it + // falls back to cached credentials during rotation mismatch + AgentIdentityUtils.CertInfo fallbackInfo = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertSame(initialInfo, fallbackInfo); + assertEquals(2, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_mtlsDisabledWithCertsPresent_logsAtMostOnce() + throws Exception { + setupValidAgentCredentialsInTempDir(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_USE_CLIENT_CERTIFICATE, "false"); + + assertFalse(AgentIdentityUtils.isMtlsDisabledLogged()); + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertTrue(AgentIdentityUtils.isMtlsDisabledLogged()); + + // Subsequent calls return null without re-triggering log message + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertTrue(AgentIdentityUtils.isMtlsDisabledLogged()); + } + + @Test + public void getAgentIdentityCertInfo_configRemovedAfterCaching_fallsBackToCachedCert() + throws Exception { + Path certPath = copyResource("agent/agent_spiffe_cert.pem", tempDir.resolve("cert.pem")); + Path keyPath = copyResource("agent/agent_spiffe_key.pem", tempDir.resolve("key.pem")); + Path configFile = tempDir.resolve("config.json"); + writeWorkloadConfig(configFile, certPath, keyPath); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toString()); + + AgentIdentityUtils.CertInfo info1 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info1); + + Files.delete(configFile); + + assertSame(info1, AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void getAgentIdentityCertInfo_wellKnownCacheThenMissingConfig_doesNotUseWellKnownCache( + @TempDir Path outsideDir) throws Exception { + setupValidAgentCredentialsInTempDir(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, null); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_USE_CLIENT_CERTIFICATE, "true"); + assertNotNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + + envProvider.setEnv( + AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, + outsideDir.resolve("missing.json").toString()); + + IOException e = assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertTrue(e.getMessage().contains("Unable to find Agent Identity certificate config or file")); + } + + @Test + public void getAgentIdentityCertInfo_configCacheThenConfigUnset_doesNotUseConfigCache() + throws Exception { + Path certPath = copyResource("agent/agent_spiffe_cert.pem", tempDir.resolve("cert.pem")); + Path keyPath = copyResource("agent/agent_spiffe_key.pem", tempDir.resolve("key.pem")); + Path configFile = tempDir.resolve("config.json"); + writeWorkloadConfig(configFile, certPath, keyPath); + Path wellKnown = Files.createDirectory(tempDir.resolve("well-known")); + AgentIdentityUtils.setWellKnownDir(wellKnown.toString() + "/"); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toString()); + assertNotNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + + // Environment variables can't change in a running JVM; this only exercises the guard. + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, null); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void getAgentIdentityCertInfo_nonAgentCachedDuringRetry_fallsBackToInitialBoundCert() + throws Exception { + setupValidAgentCredentialsInTempDir(); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + AgentIdentityUtils.CertInfo info1 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info1); + + // A mismatched key on disk makes every retry fail. + Path keyPath = tempDir.resolve("private_key.pem"); + Files.write(keyPath, mismatchedRsaKeyPem().getBytes(StandardCharsets.UTF_8)); + Files.setLastModifiedTime(keyPath, FileTime.fromMillis(System.currentTimeMillis() + 10000)); + + // While this call sleeps between retries, another caller caches a non-agent certificate. + String nonAgentCertPath = resourcePath("x509_leaf_certificate.pem"); + fakeTime.setOnSleepCallback( + () -> { + if (fakeTime.getSleepCount() == 1) { + try { + assertNull(AgentIdentityUtils.loadAndVerifyCredentials(nonAgentCertPath, null)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + }); + + assertSame(info1, AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(2, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_configModifiedMidRead_returnsVerifiedCertWithoutCaching() + throws Exception { + Path certPath = copyResource("agent/agent_spiffe_cert.pem", tempDir.resolve("cert.pem")); + Path keyPath = tempDir.resolve("key.pem"); + // The first attempt fails verification so the config change lands between attempts. + Files.write(keyPath, mismatchedRsaKeyPem().getBytes(StandardCharsets.UTF_8)); + Path configFile = tempDir.resolve("config.json"); + writeWorkloadConfig(configFile, certPath, keyPath); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toString()); + + FakeTimeService fakeTime = new FakeTimeService(); + fakeTime.setOnSleepCallback( + () -> { + if (fakeTime.getSleepCount() == 1) { + try { + copyResource("agent/agent_spiffe_key.pem", keyPath); + Files.setLastModifiedTime( + configFile, FileTime.fromMillis(System.currentTimeMillis() + 10000)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + AgentIdentityUtils.setTimeService(fakeTime); + + AgentIdentityUtils.CertInfo info1 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info1); + assertEquals(1, fakeTime.getSleepCount()); + + // Not cached, so the next refresh re-resolves from the new config and caches that result. + AgentIdentityUtils.CertInfo info2 = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info2); + assertNotSame(info1, info2); + assertSame(info2, AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void getAgentIdentityCertInfo_configDeletedMidRead_returnsVerifiedCertWithoutCaching() + throws Exception { + Path certPath = copyResource("agent/agent_spiffe_cert.pem", tempDir.resolve("cert.pem")); + Path keyPath = tempDir.resolve("key.pem"); + Files.write(keyPath, mismatchedRsaKeyPem().getBytes(StandardCharsets.UTF_8)); + Path configFile = tempDir.resolve("config.json"); + writeWorkloadConfig(configFile, certPath, keyPath); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toString()); + + FakeTimeService fakeTime = new FakeTimeService(); + fakeTime.setOnSleepCallback( + () -> { + if (fakeTime.getSleepCount() == 1) { + try { + copyResource("agent/agent_spiffe_key.pem", keyPath); + Files.delete(configFile); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + AgentIdentityUtils.setTimeService(fakeTime); + + assertNotNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(1, fakeTime.getSleepCount()); + + // Nothing was cached, so there is no cached credential to fall back to. + assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + } + + @Test + public void getAgentIdentityCertInfo_implicitDiscovery_unreadablePrivateKey_returnsNull() + throws Exception { + setupValidAgentCredentialsInTempDir(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, null); + makeUnreadable(tempDir.resolve("private_key.pem")); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void getAgentIdentityCertInfo_explicitMtls_unreadableWellKnownCert_returnsNull() + throws Exception { + setupValidAgentCredentialsInTempDir(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, null); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_USE_CLIENT_CERTIFICATE, "true"); + makeUnreadable(tempDir.resolve("certificates.pem")); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void getAgentIdentityCertInfo_configCertPathUnreadable_throwsIOExceptionNamingFile() + throws Exception { + Path certPath = copyResource("agent/agent_spiffe_cert.pem", tempDir.resolve("cert.pem")); + Path keyPath = copyResource("agent/agent_spiffe_key.pem", tempDir.resolve("key.pem")); + Path configFile = tempDir.resolve("config.json"); + writeWorkloadConfig(configFile, certPath, keyPath); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toString()); + makeUnreadable(certPath); + + IOException e = assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertTrue(e.getMessage().contains(certPath.toAbsolutePath().toString())); + } + + @Test + public void getAgentIdentityCertInfo_configInWellKnownDirMissingKeyPath_failsWithoutPolling() + throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + Path certPath = + copyResource("agent/agent_spiffe_cert.pem", tempDir.resolve("certificates.pem")); + Path configFile = tempDir.resolve("config.json"); + writeWorkloadConfig(configFile, certPath, null); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toString()); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_configInWellKnownDirMissingCertPath_failsWithoutPolling() + throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + Path keyPath = copyResource("agent/agent_spiffe_key.pem", tempDir.resolve("private_key.pem")); + Path configFile = tempDir.resolve("config.json"); + writeWorkloadConfig(configFile, null, keyPath); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toString()); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void verifyKeyPair_ecdsaLabelledEcPublicKey_returnsTrue() throws Exception { + KeyPairGenerator kpg = KeyPairGenerator.getInstance("EC"); + kpg.initialize(256); + KeyPair keyPair = kpg.generateKeyPair(); + X509Certificate cert = mock(X509Certificate.class); + when(cert.getPublicKey()) + .thenReturn(new EcdsaLabelledPublicKey((ECPublicKey) keyPair.getPublic())); + + assertTrue(AgentIdentityCertificateValidationUtils.verifyKeyPair(cert, keyPair.getPrivate())); + } + + @Test + public void getAgentIdentityCertInfo_unsupportedKeyAlgorithm_failsWithoutRetry() + throws Exception { + Path certPath = copyResource("agent/agent_spiffe_dsa_cert.pem", tempDir.resolve("cert.pem")); + Path keyPath = copyResource("agent/agent_spiffe_key.pem", tempDir.resolve("key.pem")); + Path configFile = tempDir.resolve("config.json"); + writeWorkloadConfig(configFile, certPath, keyPath); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toString()); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + IOException e = assertThrows(IOException.class, AgentIdentityUtils::getAgentIdentityCertInfo); + assertTrue(e.getMessage().contains("Unsupported key algorithm: DSA")); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_mtlsDisabled_doesNotReReadConfigOnLaterRefreshes() + throws Exception { + Path certPath = copyResource("agent/agent_spiffe_cert.pem", tempDir.resolve("cert.pem")); + Path keyPath = copyResource("agent/agent_spiffe_key.pem", tempDir.resolve("key.pem")); + Path configFile = tempDir.resolve("config.json"); + writeWorkloadConfig(configFile, certPath, keyPath); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toString()); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_USE_CLIENT_CERTIFICATE, "false"); + AtomicInteger configLookups = new AtomicInteger(); + AgentIdentityUtils.setEnvironmentProvider( + name -> { + if (AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG.equals(name)) { + configLookups.incrementAndGet(); + } + return envProvider.getEnv(name); + }); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertTrue(configLookups.get() > 0); + assertTrue(AgentIdentityUtils.isMtlsDisabledLogged()); + + configLookups.set(0); + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, configLookups.get()); + } + + @Test + public void getAgentIdentityCertInfo_mtlsDisabledWithCachedNonAgentCert_doesNotLogDisabled() + throws Exception { + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_USE_CLIENT_CERTIFICATE, "false"); + assertNull( + AgentIdentityUtils.loadAndVerifyCredentials( + resourcePath("x509_leaf_certificate.pem"), null)); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertFalse(AgentIdentityUtils.isMtlsDisabledLogged()); + } + + @Test + public void loadAndVerifyCredentials_multiCertChainResource_retainsEntireChainAndParsesLeaf() + throws Exception { + String chainPath = resourcePath("agent/agent_spiffe_cert_chain.pem"); + String keyPath = resourcePath("agent/agent_spiffe_key.pem"); + String leafPem = + new String( + Files.readAllBytes(Paths.get(resourcePath("agent/agent_spiffe_cert.pem"))), + StandardCharsets.UTF_8); + String intermediatePem = + new String( + Files.readAllBytes(Paths.get(resourcePath("x509_leaf_certificate.pem"))), + StandardCharsets.UTF_8); + + AgentIdentityUtils.CertInfo info = + AgentIdentityUtils.loadAndVerifyCredentials(chainPath, keyPath); + assertNotNull(info); + assertTrue( + AgentIdentityCertificateValidationUtils.shouldRequestBoundToken(info.getCertificate())); + assertTrue(info.getCertContent().contains(leafPem.trim())); + assertTrue(info.getCertContent().contains(intermediatePem.trim())); + } + + @Test + public void getAgentIdentityCertInfo_malformedCertificateContent_failsWithoutRetry() + throws Exception { + Path certPath = tempDir.resolve("malformed_cert.pem"); + String malformedCertPem = + "-----BEGIN CERTIFICATE-----\n" + + "bm90LWEtdmFsaWQteDUwOS1jZXJ0aWZpY2F0ZQ==\n" + + "-----END CERTIFICATE-----\n"; + Files.write(certPath, malformedCertPem.getBytes(StandardCharsets.UTF_8)); + Path keyPath = copyResource("agent/agent_spiffe_key.pem", tempDir.resolve("key.pem")); + Path configFile = tempDir.resolve("config.json"); + writeWorkloadConfig(configFile, certPath, keyPath); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, configFile.toString()); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + AgentIdentityCertificateValidationUtils.InvalidCertificateException e = + assertThrows( + AgentIdentityCertificateValidationUtils.InvalidCertificateException.class, + AgentIdentityUtils::getAgentIdentityCertInfo); + assertTrue( + e.getMessage() + .contains("Failed to parse Agent Identity certificate for bound token request.")); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void verifyKeyPair_unsupportedKeyAlgorithm_throwsUnsupportedKeyAlgorithmException() + throws Exception { + KeyPairGenerator kpg = KeyPairGenerator.getInstance("DSA"); + kpg.initialize(1024); + KeyPair keyPair = kpg.generateKeyPair(); + X509Certificate cert = mock(X509Certificate.class); + when(cert.getPublicKey()).thenReturn(keyPair.getPublic()); + + AgentIdentityCertificateValidationUtils.UnsupportedKeyAlgorithmException e = + assertThrows( + AgentIdentityCertificateValidationUtils.UnsupportedKeyAlgorithmException.class, + () -> + AgentIdentityCertificateValidationUtils.verifyKeyPair(cert, keyPair.getPrivate())); + assertTrue(e.getMessage().contains("Unsupported key algorithm: DSA")); + } + + // --- GKE credential bundle (x509.credential-bundle.private-key.pem) --- + + private Path writeGkeBundleInWellKnownDir(String certResource, String keyResource) + throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + String certPem = + new String( + Files.readAllBytes(Paths.get(resourcePath(certResource))), StandardCharsets.UTF_8); + String keyPem = + new String( + Files.readAllBytes(Paths.get(resourcePath(keyResource))), StandardCharsets.UTF_8); + Path bundle = tempDir.resolve(AgentIdentityUtils.GKE_CREDENTIAL_BUNDLE_FILE); + Files.write(bundle, (certPem + "\n" + keyPem).getBytes(StandardCharsets.UTF_8)); + return bundle; + } + + private Path writeAgentGkeBundle() throws Exception { + return writeGkeBundleInWellKnownDir( + "agent/agent_spiffe_cert.pem", "agent/agent_spiffe_key.pem"); + } + + @Test + public void getAgentIdentityCertInfo_gkeBundle_implicit_returnsCertWithoutPolling() + throws Exception { + writeAgentGkeBundle(); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + + assertNotNull(info); + assertTrue( + AgentIdentityCertificateValidationUtils.shouldRequestBoundToken(info.getCertificate())); + assertTrue(info.getCertContent().contains("BEGIN CERTIFICATE")); + assertFalse(info.getCertContent().contains("PRIVATE KEY")); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_gkeBundle_secondCallUsesCacheAndStaysBound() + throws Exception { + writeAgentGkeBundle(); + + AgentIdentityUtils.CertInfo first = AgentIdentityUtils.getAgentIdentityCertInfo(); + AgentIdentityUtils.CertInfo second = AgentIdentityUtils.getAgentIdentityCertInfo(); + + assertNotNull(first); + assertSame(first, second); + } + + @Test + public void getAgentIdentityCertInfo_gkeBundle_useClientCertFalse_returnsNull() throws Exception { + writeAgentGkeBundle(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_USE_CLIENT_CERTIFICATE, "false"); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void getAgentIdentityCertInfo_gkeBundle_enableRuntimeBoundTokenFalse_returnsNull() + throws Exception { + writeAgentGkeBundle(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "false"); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void getAgentIdentityCertInfo_gkeBundle_defaultGcloudConfigExists_returnsNull() + throws Exception { + writeAgentGkeBundle(); + Path gcloudDir = tempDir.resolve("home").resolve(".config").resolve("gcloud"); + Files.createDirectories(gcloudDir); + Files.write(gcloudDir.resolve("certificate_config.json"), "{}".getBytes()); + + // The gcloud config is the transport's mTLS certificate source, so tokens must not be bound to + // the GKE certificate. + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void getAgentIdentityCertInfo_gkeBundle_nonAgentCertificate_returnsNull() + throws Exception { + writeGkeBundleInWellKnownDir("mtls/test_cert.pem", "mtls/test_key.pem"); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void getAgentIdentityCertInfo_gkeBundleAbsent_returnsNullWithoutPolling() + throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_gkeBundle_useClientCertTrue_isDiscovered() throws Exception { + writeAgentGkeBundle(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_USE_CLIENT_CERTIFICATE, "true"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + assertNotNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_gkeBundleEmpty_returnsNullWithoutPolling() throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + Files.createFile(tempDir.resolve(AgentIdentityUtils.GKE_CREDENTIAL_BUNDLE_FILE)); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_gkeBundle_useClientCertTrue_isNotPolledFor() + throws Exception { + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_USE_CLIENT_CERTIFICATE, "true"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + + // Without GOOGLE_API_CERTIFICATE_CONFIG, no startup polling occurs when the bundle is absent. + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, fakeTime.getSleepCount()); + + // Once the bundle is written, it is found by the single up-front check. + writeAgentGkeBundle(); + assertNotNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, fakeTime.getSleepCount()); + } + + @Test + public void getAgentIdentityCertInfo_gkeBundle_takesPrecedenceOverCloudRunFilesWhenExplicit() + throws Exception { + // The transport presents the GKE bundle when no certificate config exists, so the token must be + // bound to it even if other well-known files are present. + writeAgentGkeBundle(); + copyResource("mtls/test_cert.pem", tempDir.resolve("certificates.pem")); + copyResource("mtls/test_key.pem", tempDir.resolve("private_key.pem")); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_USE_CLIENT_CERTIFICATE, "true"); + + // The GKE bundle (agent cert) wins over certificates.pem/private_key.pem (non-agent cert). + AgentIdentityUtils.CertInfo info = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(info); + } + + @Test + public void getAgentIdentityCertInfo_gkeBundleRemovedAfterCaching_fallsBackToCachedCredential() + throws Exception { + Path bundle = writeAgentGkeBundle(); + AgentIdentityUtils.CertInfo first = AgentIdentityUtils.getAgentIdentityCertInfo(); + assertNotNull(first); + + // Simulate a transient gap during rotation. + Files.delete(bundle); + + assertSame(first, AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + private void writeDefaultGcloudCertificateConfig() throws Exception { + Path gcloudDir = tempDir.resolve("home").resolve(".config").resolve("gcloud"); + Files.createDirectories(gcloudDir); + Files.write(gcloudDir.resolve("certificate_config.json"), "{}".getBytes()); + } + + @Test + public void getAgentIdentityCertInfo_gkeBundleRemovedAfterCaching_configAppears_doesNotUseCache() + throws Exception { + Path bundle = writeAgentGkeBundle(); + assertNotNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + + // The bundle is transiently missing and a certificate config now exists, so the transport + // presents the config's certificate; the cached GKE credential must not be reused. + Files.delete(bundle); + writeDefaultGcloudCertificateConfig(); + + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + } + + @Test + public void + getAgentIdentityCertInfo_gkeBundleRemovedAfterCaching_configAppears_explicit_doesNotUseCache() + throws Exception { + Path bundle = writeAgentGkeBundle(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_USE_CLIENT_CERTIFICATE, "true"); + FakeTimeService fakeTime = new FakeTimeService(); + AgentIdentityUtils.setTimeService(fakeTime); + assertNotNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + + Files.delete(bundle); + writeDefaultGcloudCertificateConfig(); + + // No other well-known files exist and the cached GKE credential is not allowed anymore. + assertNull(AgentIdentityUtils.getAgentIdentityCertInfo()); + assertEquals(0, fakeTime.getSleepCount()); + } + + private String resourcePath(String resource) throws Exception { + URL url = getClass().getClassLoader().getResource(resource); + assertNotNull(url, "Test resource " + resource + " not found"); + return Paths.get(url.toURI()).toAbsolutePath().toString(); + } + + private Path copyResource(String resource, Path target) throws Exception { + Files.copy(Paths.get(resourcePath(resource)), target, StandardCopyOption.REPLACE_EXISTING); + return target; + } + + /** Writes a workload certificate config; a null path omits that field. */ + private static void writeWorkloadConfig(Path configFile, Path certPath, Path keyPath) + throws IOException { + List fields = new ArrayList<>(); + if (certPath != null) { + fields.add("\"cert_path\": \"" + jsonPath(certPath) + "\""); + } + if (keyPath != null) { + fields.add("\"key_path\": \"" + jsonPath(keyPath) + "\""); + } + String json = "{ \"cert_configs\": { \"workload\": { " + String.join(", ", fields) + " } } }"; + Files.write(configFile, json.getBytes(StandardCharsets.UTF_8)); + } + + private static String jsonPath(Path path) { + return path.toAbsolutePath().toString().replace("\\", "\\\\"); + } + + private static String mismatchedRsaKeyPem() throws Exception { + KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); + kpg.initialize(2048); + PrivateKey key = kpg.generateKeyPair().getPrivate(); + return "-----BEGIN PRIVATE KEY-----\n" + + java.util.Base64.getEncoder().encodeToString(key.getEncoded()) + + "\n-----END PRIVATE KEY-----\n"; + } + + /** Removes all permissions; skips the test where that can't make the file unreadable. */ + private static void makeUnreadable(Path path) throws IOException { + Assumptions.assumeTrue( + FileSystems.getDefault().supportedFileAttributeViews().contains("posix"), + "POSIX file permissions are required"); + Files.setPosixFilePermissions(path, PosixFilePermissions.fromString("---------")); + Assumptions.assumeFalse(Files.isReadable(path), "Current user can read any file (e.g. root)"); + } + + /** An EC public key that reports "ECDSA" as its algorithm, as some providers do. */ + private static final class EcdsaLabelledPublicKey implements ECPublicKey { + private static final long serialVersionUID = 1L; + private final ECPublicKey delegate; + + EcdsaLabelledPublicKey(ECPublicKey delegate) { + this.delegate = delegate; + } + + @Override + public String getAlgorithm() { + return "ECDSA"; + } + + @Override + public String getFormat() { + return delegate.getFormat(); + } + + @Override + public byte[] getEncoded() { + return delegate.getEncoded(); + } + + @Override + public ECPoint getW() { + return delegate.getW(); + } + + @Override + public ECParameterSpec getParams() { + return delegate.getParams(); + } + } + + private static class FakeTimeService implements AgentIdentityUtils.TimeService { + private final AtomicLong currentTime = new AtomicLong(0); + private final AtomicInteger sleepCount = new AtomicInteger(0); + private Runnable onSleepCallback; + + void setOnSleepCallback(Runnable callback) { + this.onSleepCallback = callback; + } + + public long currentTimeMillis() { + return currentTime.get(); + } + + @Override + public void sleep(long millis) throws InterruptedException { + sleepCount.incrementAndGet(); + currentTime.addAndGet(millis); + if (onSleepCallback != null) { + onSleepCallback.run(); + } + } + + int getSleepCount() { + return sleepCount.get(); + } + } +} diff --git a/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/ComputeEngineCredentialsTest.java b/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/ComputeEngineCredentialsTest.java index 17184032e94a..bc589082af2b 100644 --- a/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/ComputeEngineCredentialsTest.java +++ b/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/ComputeEngineCredentialsTest.java @@ -48,6 +48,7 @@ import com.google.api.client.http.HttpTransport; import com.google.api.client.http.LowLevelHttpRequest; import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.json.GenericJson; import com.google.api.client.json.webtoken.JsonWebToken.Payload; import com.google.api.client.testing.http.MockLowLevelHttpRequest; import com.google.api.client.testing.http.MockLowLevelHttpResponse; @@ -64,24 +65,54 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Queue; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; /** Test case for {@link ComputeEngineCredentials}. */ class ComputeEngineCredentialsTest extends BaseSerializationTest { private static final URI CALL_URI = URI.create("http://googleapis.com/testapi/v1/foo"); + private TestEnvironmentProvider envProvider; + @TempDir private Path tempDir; + + @BeforeEach + void setUp() throws IOException { + envProvider = new TestEnvironmentProvider(); + // Inject our test environment provider into AgentIdentityUtils + AgentIdentityUtils.setEnvironmentProvider(envProvider); + + // Speed up polling in tests by using a no-op sleep service + AgentIdentityUtils.setTimeService(millis -> {}); + // Opt out of bound tokens by default in tests to avoid polling delays + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "false"); + } + + @AfterEach + void tearDown() { + // Reset the mocks + AgentIdentityUtils.resetForTest(); + } + private static final String TOKEN_URL = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; @@ -496,7 +527,8 @@ void toString_explicit_containsFields() { new MockMetadataServerTransportFactory(); String expectedToString = String.format( - "ComputeEngineCredentials{quotaProjectId=%s, universeDomain=%s, isExplicitUniverseDomain=%s, transportFactoryClassName=%s, scopes=%s}", + "ComputeEngineCredentials{quotaProjectId=%s, universeDomain=%s," + + " isExplicitUniverseDomain=%s, transportFactoryClassName=%s, scopes=%s}", "some-project", "some-domain", true, @@ -1236,6 +1268,247 @@ InputStream readStream(File file) throws FileNotFoundException { assertEquals(1, transportFactory.transport.getRequestCount()); } + @Test + void refreshAccessToken_agentConfigMissingFile_throws() throws IOException { + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "true"); + envProvider.setEnv( + AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, + tempDir.resolve("missing_config.json").toAbsolutePath().toString()); + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + AgentIdentityUtils.setTimeService(millis -> {}); + MockMetadataServerTransportFactory transportFactory = new MockMetadataServerTransportFactory(); + transportFactory.transport.setServiceAccountEmail(SA_CLIENT_EMAIL); + ComputeEngineCredentials credentials = + ComputeEngineCredentials.newBuilder().setHttpTransportFactory(transportFactory).build(); + IOException e = assertThrows(IOException.class, credentials::refreshAccessToken); + assertTrue( + e.getMessage() + .contains( + "Unable to find Agent Identity certificate config or file for bound token request" + + " after multiple retries.")); + } + + private void setupCertAndKeyConfig() throws IOException { + Path certSource = null; + try { + certSource = + Paths.get( + ComputeEngineCredentialsTest.class + .getResource("/agent/agent_spiffe_cert.pem") + .toURI()); + } catch (URISyntaxException e) { + throw new IOException("Failed to load test resource", e); + } + Path certTarget = tempDir.resolve("certificates.pem"); + Files.copy(certSource, certTarget, StandardCopyOption.REPLACE_EXISTING); + + Path keySource = null; + try { + keySource = + Paths.get( + ComputeEngineCredentialsTest.class + .getResource("/agent/agent_spiffe_key.pem") + .toURI()); + } catch (URISyntaxException e) { + throw new IOException("Failed to load test resource", e); + } + Path keyTarget = tempDir.resolve("private_key.pem"); + Files.copy(keySource, keyTarget, StandardCopyOption.REPLACE_EXISTING); + + Path configPath = tempDir.resolve("config.json"); + Map workload = new HashMap<>(); + workload.put("cert_path", certTarget.toAbsolutePath().toString()); + workload.put("key_path", keyTarget.toAbsolutePath().toString()); + + Map certConfigs = new HashMap<>(); + certConfigs.put("workload", workload); + + Map config = new HashMap<>(); + config.put("cert_configs", certConfigs); + + String configContent = OAuth2Utils.JSON_FACTORY.toString(config); + Files.write(configPath, configContent.getBytes(StandardCharsets.UTF_8)); + envProvider.setEnv("GOOGLE_API_CERTIFICATE_CONFIG", configPath.toAbsolutePath().toString()); + } + + @Test + void refreshAccessToken_withValidCertAndKey_requestsBoundToken() throws IOException { + setupCertAndKeyConfig(); + envProvider.setEnv( + AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "true"); // Enable bound token + MockMetadataServerTransportFactory transportFactory = new MockMetadataServerTransportFactory(); + transportFactory.transport.setServiceAccountEmail(SA_CLIENT_EMAIL); + + ComputeEngineCredentials credentials = + ComputeEngineCredentials.newBuilder() + .setHttpTransportFactory(transportFactory) + .setScopes(SCOPES) + .build(); + AccessToken token = credentials.refreshAccessToken(); + + assertNotNull(token); + MockLowLevelHttpRequest request = transportFactory.transport.getRequest(); + assertEquals("POST", transportFactory.transport.getRequestMethod()); + assertEquals("application/json", request.getContentType()); + assertTrue(request.getUrl().contains("scopes=foo,bar")); + String body = request.getContentAsString(); + GenericJson bodyJson = OAuth2Utils.JSON_FACTORY.fromString(body, GenericJson.class); + String expectedCert = + new String(Files.readAllBytes(tempDir.resolve("certificates.pem")), StandardCharsets.UTF_8) + .trim(); + assertEquals(expectedCert, ((String) bodyJson.get("certificate_chain")).trim()); + } + + @Test + void idTokenWithAudience_withValidCertAndKey_requestsBoundToken() throws IOException { + setupCertAndKeyConfig(); + envProvider.setEnv( + AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "true"); // Enable bound token + MockMetadataServerTransportFactory transportFactory = new MockMetadataServerTransportFactory(); + transportFactory.transport.setServiceAccountEmail(SA_CLIENT_EMAIL); + transportFactory.transport.setIdToken(STANDARD_ID_TOKEN); + + ComputeEngineCredentials credentials = + ComputeEngineCredentials.newBuilder().setHttpTransportFactory(transportFactory).build(); + IdToken token = credentials.idTokenWithAudience("https://foo.bar", null); + + assertNotNull(token); + MockLowLevelHttpRequest request = transportFactory.transport.getRequest(); + assertEquals("POST", transportFactory.transport.getRequestMethod()); + assertEquals("application/json", request.getContentType()); + assertTrue(request.getUrl().contains("audience=https://foo.bar")); + assertTrue(request.getUrl().contains("format=full")); + String body = request.getContentAsString(); + GenericJson bodyJson = OAuth2Utils.JSON_FACTORY.fromString(body, GenericJson.class); + String expectedCert = + new String(Files.readAllBytes(tempDir.resolve("certificates.pem")), StandardCharsets.UTF_8) + .trim(); + assertEquals(expectedCert, ((String) bodyJson.get("certificate_chain")).trim()); + } + + @Test + void idTokenWithAudience_disableBoundIdToken_requestsUnboundToken() throws IOException { + setupCertAndKeyConfig(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "true"); + MockMetadataServerTransportFactory transportFactory = new MockMetadataServerTransportFactory(); + transportFactory.transport.setServiceAccountEmail(SA_CLIENT_EMAIL); + transportFactory.transport.setIdToken(STANDARD_ID_TOKEN); + + ComputeEngineCredentials credentials = + ComputeEngineCredentials.newBuilder().setHttpTransportFactory(transportFactory).build(); + IdToken token = + credentials.idTokenWithAudience( + "https://foo.bar", Arrays.asList(IdTokenProvider.Option.DISABLE_BOUND_ID_TOKEN)); + + assertNotNull(token); + MockLowLevelHttpRequest request = transportFactory.transport.getRequest(); + assertEquals("GET", transportFactory.transport.getRequestMethod()); + assertTrue(request.getUrl().contains("audience=https://foo.bar")); + assertFalse(request.getUrl().contains("format=full")); + assertNull(request.getStreamingContent()); + } + + @Test + void idTokenWithAudience_disableBoundIdTokenWithFormatFull_requestsUnboundFullToken() + throws IOException { + setupCertAndKeyConfig(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "true"); + MockMetadataServerTransportFactory transportFactory = new MockMetadataServerTransportFactory(); + transportFactory.transport.setServiceAccountEmail(SA_CLIENT_EMAIL); + transportFactory.transport.setIdToken(FULL_ID_TOKEN); + + ComputeEngineCredentials credentials = + ComputeEngineCredentials.newBuilder().setHttpTransportFactory(transportFactory).build(); + credentials.idTokenWithAudience( + "https://foo.bar", + Arrays.asList( + IdTokenProvider.Option.FORMAT_FULL, IdTokenProvider.Option.DISABLE_BOUND_ID_TOKEN)); + + MockLowLevelHttpRequest request = transportFactory.transport.getRequest(); + assertEquals("GET", transportFactory.transport.getRequestMethod()); + assertTrue(request.getUrl().contains("format=full")); + assertNull(request.getStreamingContent()); + } + + @Test + void idTokenWithAudience_disableBoundIdToken_skipsCertificateLookup() throws IOException { + // The certificate config points to a missing file, which fails a bound token request. With + // DISABLE_BOUND_ID_TOKEN the certificate is never looked up, so the unbound request succeeds. + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "true"); + envProvider.setEnv( + AgentIdentityUtils.GOOGLE_API_CERTIFICATE_CONFIG, + tempDir.resolve("missing_config.json").toAbsolutePath().toString()); + AgentIdentityUtils.setWellKnownDir(tempDir.toAbsolutePath().toString() + "/"); + AgentIdentityUtils.setTimeService(millis -> {}); + MockMetadataServerTransportFactory transportFactory = new MockMetadataServerTransportFactory(); + transportFactory.transport.setServiceAccountEmail(SA_CLIENT_EMAIL); + transportFactory.transport.setIdToken(STANDARD_ID_TOKEN); + ComputeEngineCredentials credentials = + ComputeEngineCredentials.newBuilder().setHttpTransportFactory(transportFactory).build(); + + assertThrows(IOException.class, () -> credentials.idTokenWithAudience("https://foo.bar", null)); + + IdToken token = + credentials.idTokenWithAudience( + "https://foo.bar", Arrays.asList(IdTokenProvider.Option.DISABLE_BOUND_ID_TOKEN)); + assertNotNull(token); + assertEquals("GET", transportFactory.transport.getRequestMethod()); + } + + @Test + void idTokenCredentials_withDisableBoundIdTokenOption_requestsUnboundToken() throws IOException { + setupCertAndKeyConfig(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "true"); + MockMetadataServerTransportFactory transportFactory = new MockMetadataServerTransportFactory(); + transportFactory.transport.setServiceAccountEmail(SA_CLIENT_EMAIL); + transportFactory.transport.setIdToken(STANDARD_ID_TOKEN); + ComputeEngineCredentials credentials = + ComputeEngineCredentials.newBuilder().setHttpTransportFactory(transportFactory).build(); + + IdTokenCredentials idTokenCredentials = + IdTokenCredentials.newBuilder() + .setIdTokenProvider(credentials) + .setTargetAudience("https://foo.bar") + .setOptions(Arrays.asList(IdTokenProvider.Option.DISABLE_BOUND_ID_TOKEN)) + .build(); + idTokenCredentials.refresh(); + + MockLowLevelHttpRequest request = transportFactory.transport.getRequest(); + assertEquals("GET", transportFactory.transport.getRequestMethod()); + assertTrue(request.getUrl().contains("audience=https://foo.bar")); + assertNull(request.getStreamingContent()); + } + + @Test + void refreshAccessToken_boundToken404_throwsEndpointDoesNotSupportBoundTokensMessage() + throws IOException { + setupCertAndKeyConfig(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "true"); + MockMetadataServerTransportFactory transportFactory = new MockMetadataServerTransportFactory(); + transportFactory.transport.setStatusCode(HttpStatusCodes.STATUS_CODE_NOT_FOUND); + + ComputeEngineCredentials credentials = + ComputeEngineCredentials.newBuilder().setHttpTransportFactory(transportFactory).build(); + IOException e = assertThrows(IOException.class, credentials::refreshAccessToken); + assertTrue(e.getMessage().contains("does not support bound tokens")); + } + + @Test + void idTokenWithAudience_boundToken404_throwsEndpointDoesNotSupportBoundTokensMessage() + throws IOException { + setupCertAndKeyConfig(); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "true"); + MockMetadataServerTransportFactory transportFactory = new MockMetadataServerTransportFactory(); + transportFactory.transport.setStatusCode(HttpStatusCodes.STATUS_CODE_NOT_FOUND); + + ComputeEngineCredentials credentials = + ComputeEngineCredentials.newBuilder().setHttpTransportFactory(transportFactory).build(); + IOException e = + assertThrows( + IOException.class, () -> credentials.idTokenWithAudience("https://foo.bar", null)); + assertTrue(e.getMessage().contains("does not support bound tokens")); + } + static class MockMetadataServerTransportFactory implements HttpTransportFactory { MockMetadataServerTransport transport = diff --git a/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/DefaultCredentialsProviderTest.java b/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/DefaultCredentialsProviderTest.java index eceebe65902c..f2520b519280 100644 --- a/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/DefaultCredentialsProviderTest.java +++ b/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/DefaultCredentialsProviderTest.java @@ -68,11 +68,31 @@ import java.util.logging.LogRecord; import java.util.logging.Logger; import org.jspecify.annotations.Nullable; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; /** Test case for {@link DefaultCredentialsProvider}. */ class DefaultCredentialsProviderTest { + @BeforeEach + void setUp() { + // Isolate tests and opt out of bound tokens by default to avoid polling delays + AgentIdentityUtils.setEnvironmentProvider( + name -> { + if (AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN.equals(name)) { + return "false"; // Triggers isTokenBindingEnabled() = false + } + return null; + }); + } + + @AfterEach + void tearDown() { + // Reset to default behavior. + AgentIdentityUtils.resetForTest(); + } + private static final String USER_CLIENT_SECRET = "jakuaL9YyieakhECKL2SwZcu"; private static final String USER_CLIENT_ID = "ya29.1.AADtN_UtlxN3PuGAxrN2XQnZTVRvDyVWnYq4I6dws"; private static final String GCLOUDSDK_CLIENT_ID = diff --git a/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/IdTokenCredentialsTest.java b/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/IdTokenCredentialsTest.java index e3dcec4b520c..07a71cca1a54 100644 --- a/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/IdTokenCredentialsTest.java +++ b/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/IdTokenCredentialsTest.java @@ -34,11 +34,27 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; /** Test case for {@link IdTokenCredentials}. */ class IdTokenCredentialsTest extends BaseSerializationTest { + private TestEnvironmentProvider envProvider; + + @BeforeEach + void setUp() { + envProvider = new TestEnvironmentProvider(); + AgentIdentityUtils.setEnvironmentProvider(envProvider); + envProvider.setEnv(AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN, "false"); + } + + @AfterEach + void tearDown() { + AgentIdentityUtils.resetForTest(); + } + @Test void hashCode_equals() throws IOException { ComputeEngineCredentialsTest.MockMetadataServerTransportFactory transportFactory = diff --git a/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/LoggingTest.java b/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/LoggingTest.java index 524a312ce0c1..34e66daace9c 100644 --- a/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/LoggingTest.java +++ b/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/LoggingTest.java @@ -62,10 +62,14 @@ import com.google.gson.JsonParser; import com.google.gson.JsonSyntaxException; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,11 +82,39 @@ */ class LoggingTest { + @BeforeEach + void disableBoundTokensByDefault() { + // Opt out of bound tokens by default to avoid polling delays + AgentIdentityUtils.setEnvironmentProvider( + name -> { + if (AgentIdentityUtils.GOOGLE_API_ENABLE_RUNTIME_BOUND_TOKEN.equals(name)) { + return "false"; + } + return null; + }); + } + + private final List modifiedLoggers = new ArrayList<>(); + private final List addedAppenders = new ArrayList<>(); + + @AfterEach + void tearDown() { + AgentIdentityUtils.resetForTest(); + for (int i = 0; i < modifiedLoggers.size(); i++) { + modifiedLoggers.get(i).detachAppender(addedAppenders.get(i)); + } + modifiedLoggers.clear(); + addedAppenders.clear(); + } + private TestAppender setupTestLogger(Class clazz) { TestAppender testAppender = new TestAppender(); testAppender.start(); - Logger logger = LoggerFactory.getLogger(clazz); - ((ch.qos.logback.classic.Logger) logger).addAppender(testAppender); + ch.qos.logback.classic.Logger logger = + (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(clazz); + logger.addAppender(testAppender); + modifiedLoggers.add(logger); + addedAppenders.add(testAppender); return testAppender; } @@ -94,6 +126,11 @@ static void setup() { LoggingUtils.setEnvironmentProvider(testEnvironmentProvider); } + @AfterAll + static void tearDownAll() { + LoggingUtils.setEnvironmentProvider(SystemEnvironmentProvider.getInstance()); + } + @Test void userCredentials_getRequestMetadata_fromRefreshToken_hasAccessToken() throws IOException { TestAppender testAppender = setupTestLogger(UserCredentials.class); diff --git a/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/MockMetadataServerTransport.java b/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/MockMetadataServerTransport.java index 369ea35b857c..1f8878587c88 100644 --- a/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/MockMetadataServerTransport.java +++ b/google-auth-library-java/oauth2_http/javatests/com/google/auth/oauth2/MockMetadataServerTransport.java @@ -72,6 +72,7 @@ public class MockMetadataServerTransport extends MockHttpTransport { private boolean emptyContent; private MockLowLevelHttpRequest request; private int requestCount = 0; + private String requestMethod; public int getRequestCount() { return requestCount; @@ -128,9 +129,14 @@ public MockLowLevelHttpRequest getRequest() { return request; } + public String getRequestMethod() { + return requestMethod; + } + @Override public LowLevelHttpRequest buildRequest(String method, String url) throws IOException { requestCount++; + this.requestMethod = method; if (url.startsWith(ComputeEngineCredentials.getTokenServerEncodedUrl())) { this.request = getMockRequestForTokenEndpoint(url); return this.request; diff --git a/google-auth-library-java/oauth2_http/testresources/agent/agent_spiffe_cert.pem b/google-auth-library-java/oauth2_http/testresources/agent/agent_spiffe_cert.pem new file mode 100644 index 000000000000..885ecec9b6cf --- /dev/null +++ b/google-auth-library-java/oauth2_http/testresources/agent/agent_spiffe_cert.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDcjCCAlqgAwIBAgIUb5t9VPgT3uTrj+z47BfU8xF6sFowDQYJKoZIhvcNAQEL +BQAwIjEgMB4GA1UEAwwXVGVzdCBTUElGRkUgQ2VydGlmaWNhdGUwHhcNMjYwNTEx +MjMyODU3WhcNMzYwNTA4MjMyODU3WjAiMSAwHgYDVQQDDBdUZXN0IFNQSUZGRSBD +ZXJ0aWZpY2F0ZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJlPtSAh +hYqci8YJaPlgZWUbup4q/TGuaLdR/zWpFCevAQW7/hq29LT8WsoyuirSlbJg67io +PPuNp98e614L/88jP3wMeSuBSXGMyeExrMQ35ZbrnmD268Fl/jEZXPujmeFWmRoy +miKSkNvYLvkN0fiooyFYZdNG0/f6zT+dwJqoOYgeBl9XXDf5QjnQbhQXFQJIbxfQ +TTaGFtEXXwKCs9cd5EvwbiDuaM+F2O7txkaD4r+77AjsXWUFkBBbSUzs2b3ujZot +ZqwNCeeGML/G+5TQj3UExTtDzYpjvncEuqdomFraqXy6DmyELhyChugsz5rGx5Vt +aP3RvDkpLwUxjWcCAwEAAaOBnzCBnDAdBgNVHQ4EFgQUoS1OUx9a1hBHt0tHAefV +7QQ3JYswHwYDVR0jBBgwFoAUoS1OUx9a1hBHt0tHAefV7QQ3JYswDwYDVR0TAQH/ +BAUwAwEB/zBJBgNVHREEQjBAhj5zcGlmZmU6Ly9hZ2VudHMuZ2xvYmFsLnByb2ot +MTIzNDUuc3lzdGVtLmlkLmdvb2cvdGVzdC13b3JrbG9hZDANBgkqhkiG9w0BAQsF +AAOCAQEAfCrNuLFIlpvtDpBKD8lxj2vF4/6fbLhl/5YGGFf2uydaWLr2hTmLFrer +fzGf04hPD5UJTms0ZlvHTvapm1IikxKhkmp8GOlMvKWl/EwIDaJyJ8EaYCpkrTNs +pLR/ujgeSKnIHgm2Oql7HZxB1T25teFcLIIMd0zs69h/Sxejw+OTKnwb7san1qSy +uNvXVDPNFrGyjvBXAVyjyvs3Adz/A+GYhyaP31s+3qGUUR4/axv0M8pUVmK6D8lU +5TkO7smVELt51xaq77Nvv1r5FasJkF5CrnqwAjq1QfVJpDsuTCEriUXguxaXFbzM +Qfw7TEXp3xPVinhkDjlRfiGg6hIvNw== +-----END CERTIFICATE----- diff --git a/google-auth-library-java/oauth2_http/testresources/agent/agent_spiffe_cert_chain.pem b/google-auth-library-java/oauth2_http/testresources/agent/agent_spiffe_cert_chain.pem new file mode 100644 index 000000000000..2673b40653b7 --- /dev/null +++ b/google-auth-library-java/oauth2_http/testresources/agent/agent_spiffe_cert_chain.pem @@ -0,0 +1,40 @@ +-----BEGIN CERTIFICATE----- +MIIDcjCCAlqgAwIBAgIUb5t9VPgT3uTrj+z47BfU8xF6sFowDQYJKoZIhvcNAQEL +BQAwIjEgMB4GA1UEAwwXVGVzdCBTUElGRkUgQ2VydGlmaWNhdGUwHhcNMjYwNTEx +MjMyODU3WhcNMzYwNTA4MjMyODU3WjAiMSAwHgYDVQQDDBdUZXN0IFNQSUZGRSBD +ZXJ0aWZpY2F0ZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJlPtSAh +hYqci8YJaPlgZWUbup4q/TGuaLdR/zWpFCevAQW7/hq29LT8WsoyuirSlbJg67io +PPuNp98e614L/88jP3wMeSuBSXGMyeExrMQ35ZbrnmD268Fl/jEZXPujmeFWmRoy +miKSkNvYLvkN0fiooyFYZdNG0/f6zT+dwJqoOYgeBl9XXDf5QjnQbhQXFQJIbxfQ +TTaGFtEXXwKCs9cd5EvwbiDuaM+F2O7txkaD4r+77AjsXWUFkBBbSUzs2b3ujZot +ZqwNCeeGML/G+5TQj3UExTtDzYpjvncEuqdomFraqXy6DmyELhyChugsz5rGx5Vt +aP3RvDkpLwUxjWcCAwEAAaOBnzCBnDAdBgNVHQ4EFgQUoS1OUx9a1hBHt0tHAefV +7QQ3JYswHwYDVR0jBBgwFoAUoS1OUx9a1hBHt0tHAefV7QQ3JYswDwYDVR0TAQH/ +BAUwAwEB/zBJBgNVHREEQjBAhj5zcGlmZmU6Ly9hZ2VudHMuZ2xvYmFsLnByb2ot +MTIzNDUuc3lzdGVtLmlkLmdvb2cvdGVzdC13b3JrbG9hZDANBgkqhkiG9w0BAQsF +AAOCAQEAfCrNuLFIlpvtDpBKD8lxj2vF4/6fbLhl/5YGGFf2uydaWLr2hTmLFrer +fzGf04hPD5UJTms0ZlvHTvapm1IikxKhkmp8GOlMvKWl/EwIDaJyJ8EaYCpkrTNs +pLR/ujgeSKnIHgm2Oql7HZxB1T25teFcLIIMd0zs69h/Sxejw+OTKnwb7san1qSy +uNvXVDPNFrGyjvBXAVyjyvs3Adz/A+GYhyaP31s+3qGUUR4/axv0M8pUVmK6D8lU +5TkO7smVELt51xaq77Nvv1r5FasJkF5CrnqwAjq1QfVJpDsuTCEriUXguxaXFbzM +Qfw7TEXp3xPVinhkDjlRfiGg6hIvNw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDIzCCAgugAwIBAgIJAMfISuBQ5m+5MA0GCSqGSIb3DQEBBQUAMBUxEzARBgNV +BAMTCnVuaXQtdGVzdHMwHhcNMTExMjA2MTYyNjAyWhcNMjExMjAzMTYyNjAyWjAV +MRMwEQYDVQQDEwp1bml0LXRlc3RzMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEA4ej0p7bQ7L/r4rVGUz9RN4VQWoej1Bg1mYWIDYslvKrk1gpj7wZgkdmM +7oVK2OfgrSj/FCTkInKPqaCR0gD7K80q+mLBrN3PUkDrJQZpvRZIff3/xmVU1Wer +uQLFJjnFb2dqu0s/FY/2kWiJtBCakXvXEOb7zfbINuayL+MSsCGSdVYsSliS5qQp +gyDap+8b5fpXZVJkq92hrcNtbkg7hCYUJczt8n9hcCTJCfUpApvaFQ18pe+zpyl4 ++WzkP66I28hniMQyUlA1hBiskT7qiouq0m8IOodhv2fagSZKjOTTU2xkSBc//fy3 +ZpsL7WqgsZS7Q+0VRK8gKfqkxg5OYQIDAQABo3YwdDAdBgNVHQ4EFgQU2RQ8yO+O +gN8oVW2SW7RLrfYd9jEwRQYDVR0jBD4wPIAU2RQ8yO+OgN8oVW2SW7RLrfYd9jGh +GaQXMBUxEzARBgNVBAMTCnVuaXQtdGVzdHOCCQDHyErgUOZvuTAMBgNVHRMEBTAD +AQH/MA0GCSqGSIb3DQEBBQUAA4IBAQBRv+M/6+FiVu7KXNjFI5pSN17OcW5QUtPr +odJMlWrJBtynn/TA1oJlYu3yV5clc/71Vr/AxuX5xGP+IXL32YDF9lTUJXG/uUGk ++JETpKmQviPbRsvzYhz4pf6ZIOZMc3/GIcNq92ECbseGO+yAgyWUVKMmZM0HqXC9 +ovNslqe0M8C1sLm1zAR5z/h/litE7/8O2ietija3Q/qtl2TOXJdCA6sgjJX2WUql +ybrC55ct18NKf3qhpcEkGQvFU40rVYApJpi98DiZPYFdx1oBDp/f4uZ3ojpxRVFT +cDwcJLfNRCPUhormsY7fDS9xSyThiHsW9mjJYdcaKQkwYZ0F11yB +-----END CERTIFICATE----- diff --git a/google-auth-library-java/oauth2_http/testresources/agent/agent_spiffe_dsa_cert.pem b/google-auth-library-java/oauth2_http/testresources/agent/agent_spiffe_dsa_cert.pem new file mode 100644 index 000000000000..c265daf51152 --- /dev/null +++ b/google-auth-library-java/oauth2_http/testresources/agent/agent_spiffe_dsa_cert.pem @@ -0,0 +1,28 @@ +-----BEGIN CERTIFICATE----- +MIIE0zCCBICgAwIBAgIUWCi6wrWkagd68xNCmHNvDFP1wM4wCwYJYIZIAWUDBAMC +MCYxJDAiBgNVBAMMG1Rlc3QgU1BJRkZFIERTQSBDZXJ0aWZpY2F0ZTAeFw0yNjA5 +MjYwMzU1NTJaFw0zNjA5MjMwMzU1NTJaMCYxJDAiBgNVBAMMG1Rlc3QgU1BJRkZF +IERTQSBDZXJ0aWZpY2F0ZTCCA0IwggI1BgcqhkjOOAQBMIICKAKCAQEAi8+zfl/M +BnfjpGEB/+d1yzOepM7URTjVENircKJMMWm4WVvbdC86oaGsLjzWF+s7EKE5p3it +J9Yq3gUT+y7iOBrQRTJWecaiXCzuaVZi6gufk5wNd0jMpEcc6xySB9+a0RgfXldW +O7ROCQpK84zPoCLwmK8JsE3Bor6ccIaf29/mZNZdFYKDGei87Zkw8894EI1gRiLV +BoB01ZGGzyHRVi70U7SBD7YV0pHqa4sIYFLy6NrBBJz2IMG0vNVDcobfJt8W5NzN +3xbSjE00Ek5/L7jSI8KhPabfAV9cdTmWgt8xhjHs4MVSGMOBXPhYLfdumRdgeYke +dmthW+qAvknbywIdAK5lwn3zIMn65pIbME54vAf+yypNgDRleJ7KzA0CggEATbWw +qeqLhsIUB1ZXHdpIi5T1Fq4mu1DbI0fPlivw8wrodIJXUMVEeyH4rohoJ83c9uEw +5G7LOwfFaNkoUvSqg4tZ2+2v6wYgDnDbs9mXwsDtguxfy2+H6wQo+v+At8BTTaay +IrcXsZsFv/tCmeylulRBCzMAaUvUhcoyhFlaQ91pJPHpkiCX/u4gxbsdBwZIe8Fu +lnz0xi4NBCIH4BjAPaEiEylwZmWS+C9TDfvFp4Tm3IAnevoMnxcF4UC2UK5vYbef +Rayf2sXVeMfEgDRfPkbCENTzysZIM2raKyA05h9+4x9YaiLemM37j3eML8kEwF+x +QZKKzp6YWJabqSylEQOCAQUAAoIBABZax0scLZq7/55jGT49ku36f4XRqH75gsUg +++UFZBbjdOeUKm6i/qNHTYABmDL1LGYnjxTbUFankhnYM4wpPvTFS9/NPYLuwkdi +A3wRHro/18BlZqS+ugTDz5+5PpAG8qVOOktHAPs6V/0fb0XPtoA3IMr1/RNVu/1E +kKEs80pjd9gme7XjGRDNUCuo51EXinuMTbfuwa2Y37dDkp7+W6gtkDr0JiNgY+Od +ga6gVhxREA3qnTjrWlBmUy+1/LQLIqU4DGza3FW2+auo7G+s4/qj78XQeHkVKgZ+ +QZYYHZ936sXCmC4aGLioUJY4eezWgZy3N4Ymoxe8+vAJiPCcF/KjgZ8wgZwwHQYD +VR0OBBYEFALBYVM0fPAA3AUku9yMCJcjAfpkMB8GA1UdIwQYMBaAFALBYVM0fPAA +3AUku9yMCJcjAfpkMA8GA1UdEwEB/wQFMAMBAf8wSQYDVR0RBEIwQIY+c3BpZmZl +Oi8vYWdlbnRzLmdsb2JhbC5wcm9qLTEyMzQ1LnN5c3RlbS5pZC5nb29nL3Rlc3Qt +d29ya2xvYWQwCwYJYIZIAWUDBAMCA0AAMD0CHG1Y0GRAhiBgo5nfdS9+0dUj8eTi +Bs+r4exRAAUCHQCP2U3KqooDvlDngASQwZPq35wrvM1Rs6Dvc9P8 +-----END CERTIFICATE----- diff --git a/google-auth-library-java/oauth2_http/testresources/agent/agent_spiffe_key.pem b/google-auth-library-java/oauth2_http/testresources/agent/agent_spiffe_key.pem new file mode 100644 index 000000000000..b136c31e3db7 --- /dev/null +++ b/google-auth-library-java/oauth2_http/testresources/agent/agent_spiffe_key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCZT7UgIYWKnIvG +CWj5YGVlG7qeKv0xrmi3Uf81qRQnrwEFu/4atvS0/FrKMroq0pWyYOu4qDz7jaff +HuteC//PIz98DHkrgUlxjMnhMazEN+WW655g9uvBZf4xGVz7o5nhVpkaMpoikpDb +2C75DdH4qKMhWGXTRtP3+s0/ncCaqDmIHgZfV1w3+UI50G4UFxUCSG8X0E02hhbR +F18CgrPXHeRL8G4g7mjPhdju7cZGg+K/u+wI7F1lBZAQW0lM7Nm97o2aLWasDQnn +hjC/xvuU0I91BMU7Q82KY753BLqnaJha2ql8ug5shC4cgoboLM+axseVbWj90bw5 +KS8FMY1nAgMBAAECggEAA/g2Eq6bhEYr/lHL2u0kIvT2JSNGBAb/WIPLs/cWE2le +GujOGjrGskSSEaGa0JuiqAg6m9p+hCFPIyFShQOSUsNkYOqUkDJPe8+vG62xsTDx +uMsoAijsKMlI+bqtsPXm+MuWHw4Hww5nDsvI4Sz6iB6bS4E8JB2cEaDEtx/do7aq +5XHJYoy0kO/l87n1UlIq8Pi6yW36pQKKEK+Dm5q+QmgJIeNLFp0hb5r7/fLFENTD +CZO0GvMdmtTkTwyPB1mvFjP6mNNAZcO6j9eaWNCxFhPH+MNWtF+694wgPOQIFKL3 +n4FPmWUS7vGJjNI55Ff2tTHrbWcMuf4RwXbVIG+v1QKBgQDPnNEs2TRrMglFtT6J +wDbICK8AQbBAPFjtZUGxNnsCOdiprZXshZJ7Yq8lokDlsU+5l0YTzRj2v1IcEL49 +oy8WeQfbcOZTGgf+Qo50ZiLhziCqNTlxiXMp3Kg5h4cjxUz2tsF2Nmzn75411uPa +Q/SKHCzfz4uUtHJla8tHENuzdQKBgQC9CwOleHwdS5D1jzBpb/+rvc0vcyYeihZe +v2ao2ItEZZpDyZoou3HiuTe8b9jpg4iS1PtBe6ck/99dBA3CSCF6VCS+++fc+HOz +IyPZjfg8qQ/88XMIEJy8eAa2OFT/EwK3AeZvIuXVumUHI2R8AfkVB0OfjSVZ7hnI +p1ndQ09t6wKBgHDiQVHzX+8RK719SN25Z4/oOM8Y6G5k4a1iqw9iIgwZy9amjagn +EHiKNdVunX7GpCSzPeUyVWqEqG6eI/J7sfS0JjOI9ZMlykbThYWAq2K/oz8o5Wz4 +YWfXlJiDOlWWx7w1rodKHHkX7pwzlXxuCp61pyiiPrDCVJkUvViMsAipAoGAQHda +FfqhcKgNVgAvhTVBXgLKzwyYij+S41qoGppF29w+IDHG1W8epi99d1A5C2DkmRXy +XOFbHX34YNL6Ei/g4sOBCHQFHNDJO+SW3CDS73TD1AFOtghcOtU/jLJnIdkMyvXl +7C5dbGY0/5stMDDIDUi94dITU7ijqE6RkafblWMCgYEAxwFH2JJn8HyU30PVrIFh +AsviZ3BYrCbYFlkZpelCHzHc5imyR9euLwXHIPS0vTE5V8GzkPEgW3DIOFMmBwe2 +8M/jGi/RQCnD7Hh0C+Y0vtUyF8z88lvN5ac6lU9UpGVBBkd8HEsq5ukL3PlnW4B0 +uRGJOa6FU01LIEMoCL5LP28= +-----END PRIVATE KEY----- diff --git a/google-auth-library-java/oauth2_http/testresources/mtls/test_chain_cert.pem b/google-auth-library-java/oauth2_http/testresources/mtls/test_chain_cert.pem new file mode 100644 index 000000000000..5465eb9a51b1 --- /dev/null +++ b/google-auth-library-java/oauth2_http/testresources/mtls/test_chain_cert.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIB2TCCAX+gAwIBAgIUNavUXQ/rulG+3syxU0MrxecWQGQwCgYIKoZIzj0EAwIw +HzEdMBsGA1UEAwwUVGVzdCBJbnRlcm1lZGlhdGUgQ0EwIBcNMjYxMDA5MTUxODM3 +WhgPMjEyNjA5MTUxNTE4MzdaMBQxEjAQBgNVBAMMCVRlc3QgTGVhZjBZMBMGByqG +SM49AgEGCCqGSM49AwEHA0IABFFirHZTwx/ld55xeocgpp0rQME9O9ATEonLHecx +RYlG8N0eMNTC4PyIVCQPfHnrI167YpwCk2nyh+HS8wl3UrOjgaEwgZ4wDAYDVR0T +AQH/BAIwADAOBgNVHQ8BAf8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUHAwIwKQYD +VR0RBCIwIIYec3BpZmZlOi8vZXhhbXBsZS50ZXN0L3dvcmtsb2FkMB0GA1UdDgQW +BBSXEYG9sUpIGjEVcNnv5z6R/Y0UGjAfBgNVHSMEGDAWgBQtGnDllv2kiVva8/vP +KnPkeOYblTAKBggqhkjOPQQDAgNIADBFAiBFg0vUmv7/I2KMcnIlDMpxwQhl7KUH +fTvnJUu3TlCq3wIhAMpR49B2EVADdJTP630dUMbxQl7i0KoSinyZe+eXbGo7 +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIBoTCCAUagAwIBAgIULt67iL18WCU7vF52uolgyC26bL8wCgYIKoZIzj0EAwIw +FzEVMBMGA1UEAwwMVGVzdCBSb290IENBMCAXDTI2MTAwOTE1MTgzN1oYDzIxMjYw +OTE1MTUxODM3WjAfMR0wGwYDVQQDDBRUZXN0IEludGVybWVkaWF0ZSBDQTBZMBMG +ByqGSM49AgEGCCqGSM49AwEHA0IABPOMvOWtJYOV6I9Sp6C7gzA3iOi3VCz9kZQ2 +iW86iUDpk+S5jso0pP3gJy8hdJBLGkcmm9L+x4CaRcCSePsQ/vyjZjBkMBIGA1Ud +EwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBQtGnDllv2k +iVva8/vPKnPkeOYblTAfBgNVHSMEGDAWgBT8Ay0RXQdWDWCi9PzNTwvXx+W+KzAK +BggqhkjOPQQDAgNJADBGAiEA/K0vk+MiOpF0G0Rrpw2tpi+DtIXUlkPLSTzXVGMn +cGgCIQChgI06W0im1y5pz3ymwTewRSF7MDlCgKt+pALGcZv1Gw== +-----END CERTIFICATE----- diff --git a/google-auth-library-java/oauth2_http/testresources/mtls/test_chain_key.pem b/google-auth-library-java/oauth2_http/testresources/mtls/test_chain_key.pem new file mode 100644 index 000000000000..37ef3f333f18 --- /dev/null +++ b/google-auth-library-java/oauth2_http/testresources/mtls/test_chain_key.pem @@ -0,0 +1,5 @@ +-----BEGIN PRIVATE KEY----- +MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgnkUdqWqfKVO0qTH6 +WByFOh5OjqL9NOuQebS1JhEL7SChRANCAARRYqx2U8Mf5XeecXqHIKadK0DBPTvQ +ExKJyx3nMUWJRvDdHjDUwuD8iFQkD3x56yNeu2KcApNp8ofh0vMJd1Kz +-----END PRIVATE KEY----- diff --git a/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java b/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java index fab73a55dccf..d31a76619725 100644 --- a/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java +++ b/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/ChannelPool.java @@ -31,6 +31,7 @@ import com.google.api.core.InternalApi; import com.google.api.gax.core.FixedExecutorProvider; +import com.google.api.gax.rpc.mtls.CertificateRotationTracker; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -53,6 +54,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; @@ -72,18 +74,28 @@ @NullMarked class ChannelPool extends ManagedChannel { static final String CHANNEL_POOL_CONSECUTIVE_RESIZING_WARNING = - "The gRPC ChannelPool used in the client has been flagged to be repeatedly resizing (5+ times). See https://github.com/googleapis/google-cloud-java/blob/main/docs/grpc_channel_pool_guide.md for more information about this behavior."; + "The gRPC ChannelPool used in the client has been flagged to be repeatedly resizing (5+" + + " times). See" + + " https://github.com/googleapis/google-cloud-java/blob/main/docs/grpc_channel_pool_guide.md" + + " for more information about this behavior."; @VisibleForTesting static final Logger LOG = Logger.getLogger(ChannelPool.class.getName()); private static final java.time.Duration REFRESH_PERIOD = java.time.Duration.ofMinutes(50); private final ChannelPoolSettings settings; private final ChannelFactory channelFactory; private final FixedExecutorProvider backgroundExecutorProvider; + private final String workloadCertPath; private @Nullable ScheduledFuture refreshFuture = null; private @Nullable ScheduledFuture resizeFuture = null; + private final CertificateRotationTracker rotationTracker; private final Object entryWriteLock = new Object(); + + // Guarded by entryWriteLock. + private boolean isShutdown = false; + + private final AtomicLong generation = new AtomicLong(0); @VisibleForTesting final AtomicReference> entries = new AtomicReference<>(); private final AtomicInteger indexTicker = new AtomicInteger(); private final String authority; @@ -100,14 +112,15 @@ class ChannelPool extends ManagedChannel { static ChannelPool create( ChannelPoolSettings settings, ChannelFactory channelFactory, - @Nullable ScheduledExecutorService backgroundExecutor) + @Nullable ScheduledExecutorService backgroundExecutor, + @Nullable String workloadCertPath) throws IOException { FixedExecutorProvider executorProvider = backgroundExecutor == null ? FixedExecutorProvider.create(Executors.newSingleThreadScheduledExecutor(), true) : FixedExecutorProvider.create(backgroundExecutor, false); - return new ChannelPool(settings, channelFactory, executorProvider); + return new ChannelPool(settings, channelFactory, executorProvider, workloadCertPath); } /** @@ -121,11 +134,14 @@ static ChannelPool create( ChannelPool( ChannelPoolSettings settings, ChannelFactory channelFactory, - FixedExecutorProvider executorProvider) + FixedExecutorProvider executorProvider, + @Nullable String workloadCertPath) throws IOException { this.settings = settings; this.channelFactory = channelFactory; this.backgroundExecutorProvider = executorProvider; + this.workloadCertPath = workloadCertPath; + this.rotationTracker = new CertificateRotationTracker(workloadCertPath); ImmutableList.Builder initialListBuilder = ImmutableList.builder(); @@ -187,7 +203,8 @@ public ManagedChannel shutdown() { // Resize and refresh tasks can block on channel priming. We don't need // to wait for the channels to be ready since we're shutting down the - // pool. Allowing interrupt to speed it up. + // pool. Allowing interrupt to speed it up. This is done before acquiring + // entryWriteLock, which a running refresh or resize holds. if (resizeFuture != null) { resizeFuture.cancel(true); } @@ -195,9 +212,12 @@ public ManagedChannel shutdown() { refreshFuture.cancel(true); } - List localEntries = entries.get(); - for (Entry entry : localEntries) { - entry.channel.shutdown(); + synchronized (entryWriteLock) { + isShutdown = true; + List localEntries = entries.get(); + for (Entry entry : localEntries) { + entry.channel.shutdown(); + } } if (backgroundExecutorProvider.shouldAutoClose()) { @@ -210,6 +230,11 @@ public ManagedChannel shutdown() { /** {@inheritDoc} */ @Override public boolean isShutdown() { + synchronized (entryWriteLock) { + if (isShutdown) { + return true; + } + } List localEntries = entries.get(); for (Entry entry : localEntries) { if (!entry.channel.isShutdown()) { @@ -236,6 +261,8 @@ public boolean isTerminated() { public ManagedChannel shutdownNow() { LOG.fine("Initiating immediate shutdown due to explicit request"); + // Cancel before acquiring entryWriteLock, which a running refresh or resize holds, so that + // they are interrupted instead of delaying shutdown. if (resizeFuture != null) { resizeFuture.cancel(true); } @@ -243,9 +270,12 @@ public ManagedChannel shutdownNow() { refreshFuture.cancel(true); } - List localEntries = entries.get(); - for (Entry entry : localEntries) { - entry.channel.shutdownNow(); + synchronized (entryWriteLock) { + isShutdown = true; + List localEntries = entries.get(); + for (Entry entry : localEntries) { + entry.channel.shutdownNow(); + } } if (backgroundExecutorProvider.shouldAutoClose()) { @@ -272,6 +302,11 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedE private void resizeSafely() { try { synchronized (entryWriteLock) { + // shutdown() cancels this task, but cancel(true) does not interrupt a run that is already + // waiting for entryWriteLock, so it must not create channels once the pool is shut down. + if (isShutdown) { + return; + } resize(); } } catch (Exception e) { @@ -411,7 +446,7 @@ private void expand(int desiredSize) { for (int i = 0; i < desiredSize - localEntries.size(); i++) { try { newEntries.add(new Entry(channelFactory.createSingleChannel())); - } catch (IOException e) { + } catch (Exception e) { LOG.log(Level.WARNING, "Failed to add channel", e); } } @@ -419,23 +454,81 @@ private void expand(int desiredSize) { entries.set(newEntries.build()); } + /** + * Periodically refreshes all channels when {@link + * ChannelPoolSettings#isPreemptiveRefreshEnabled()} is enabled (to mitigate hourly GFE + * disconnects). This applies to all channels even when {@code workloadCertPath == null}. If + * {@code workloadCertPath} is configured, the refresh is skipped while the certificate file is + * unreadable or mid-write on disk. + * + *

    If the certificate on disk differs from the active one, this refresh is treated like a + * rotation refresh: channels that fail to be recreated are dropped (and the pool refilled), so + * the switch is completed and the generation advanced only once every channel left in the pool + * uses the new certificate. Otherwise a channel that fails to be recreated keeps its slot, and + * the generation is not advanced, so that a periodic refresh without a rotation does not make + * in-flight {@code UNAUTHENTICATED} failures eligible for a rotation retry. + */ private void refreshSafely() { try { - refresh(); + // See refresh(): a generation change while waiting for the lock means a concurrent refresh + // just recreated every channel on a new certificate. + long generationBeforeLock = generation.get(); + synchronized (entryWriteLock) { + if (workloadCertPath == null) { + refreshAll(); + return; + } + if (generation.get() != generationBeforeLock) { + LOG.fine( + "Skipping pre-emptive channel refresh: channels were just recreated by a concurrent" + + " refresh"); + return; + } + String currentDiskFingerprint = rotationTracker.readDiskFingerprint(); + if (currentDiskFingerprint.isEmpty()) { + // The configured certificate could not be read, which normally means it is being + // rewritten during a rotation. Skip this refresh rather than recreate channels from a + // partially written certificate; the next periodic or reactive refresh recreates them. + LOG.fine( + "Skipping pre-emptive channel refresh: the workload certificate could not be read;" + + " channels will be recreated on the next refresh"); + return; + } + boolean certChanged = !rotationTracker.isAlreadyActive(currentDiskFingerprint); + if (refreshAll(/* dropUnrefreshedChannels= */ certChanged) && certChanged) { + completeCertificateSwitch(currentDiskFingerprint); + } + } } catch (Exception e) { - LOG.log(Level.WARNING, "Failed to pre-emptively refresh channnels", e); + LOG.log(Level.WARNING, "Failed to pre-emptively refresh channels", e); } } + @VisibleForTesting + void invalidateDiskFingerprintCache() { + rotationTracker.invalidateCache(); + } + + boolean shouldRefresh() { + return rotationTracker.shouldRefresh(); + } + /** * Replace all of the channels in the channel pool with fresh ones. This is meant to mitigate the - * hourly GFE disconnects by giving clients the ability to prime the channel on reconnect. + * hourly GFE disconnects by giving clients the ability to prime the channel on reconnect, and to + * pick up a rotated mTLS workload certificate. * - *

    This is done on a best effort basis. If the replacement channel fails to construct, the old - * channel will continue to be used. + *

    This is done on a best effort basis. When no workload certificate is configured, a channel + * whose replacement fails to construct continues to be used. When a workload certificate is + * configured, a channel whose replacement fails to construct is dropped so that no traffic is + * routed to a channel using the old certificate; if no replacement can be constructed at all, the + * pool is left unchanged. */ @InternalApi("Visible for testing") void refresh() { + // A generation change while waiting for the lock means a concurrent refresh already switched + // every channel to a new certificate, so there is no need to read the certificate again. + long generationBeforeLock = generation.get(); // Note: synchronization is necessary in case refresh is called concurrently: // - thread1 fails to replace a single entry // - thread2 succeeds replacing an entry @@ -443,28 +536,179 @@ void refresh() { // - then thread2 will shut down channel that thread1 will put back into circulation (after it // replaces the list) synchronized (entryWriteLock) { + if (isShutdown) { + return; + } + if (workloadCertPath == null) { + refreshAll(); + return; + } + if (generation.get() != generationBeforeLock) { + LOG.fine( + "Channel pool was already refreshed by a concurrent thread, skipping duplicate" + + " refresh"); + return; + } + String currentDiskFingerprint = rotationTracker.readDiskFingerprint(); + if (currentDiskFingerprint.isEmpty()) { + return; + } + + // Double-check fingerprint inside the lock + if (rotationTracker.isAlreadyActive(currentDiskFingerprint)) { + LOG.fine( + "Channel pool was already refreshed by a concurrent thread, skipping duplicate" + + " refresh"); + return; + } + + // Drop any channel that fails to refresh so that no traffic is routed to the old certificate. + if (refreshAll(/* dropUnrefreshedChannels= */ true)) { + completeCertificateSwitch(currentDiskFingerprint); + } + } + } + + /** + * Records that every channel in the pool now uses the certificate with the given fingerprint. + * Must be called while holding {@code entryWriteLock}, after the channels have been swapped. + * + *

    The generation is incremented before the fingerprint is marked active, so that a concurrent + * failing RPC that no longer sees a pending rotation ({@link #shouldRefresh()} is {@code false}) + * is guaranteed to see the new generation and be retried on the new channels. + */ + private void completeCertificateSwitch(String newFingerprint) { + generation.incrementAndGet(); + rotationTracker.markRefreshed(newFingerprint); + LOG.fine("Channel pool switched to certificate with fingerprint: " + newFingerprint); + } + + @InternalApi("Visible for testing") + @Nullable String getWorkloadCertPath() { + return workloadCertPath; + } + + @InternalApi("Visible for testing") + boolean refreshAll() { + return refreshAll(/* dropUnrefreshedChannels= */ false); + } + + /** + * Replaces the channels in the pool with freshly created ones. + * + * @param dropUnrefreshedChannels if {@code false}, a channel that fails to be recreated keeps its + * slot in the pool. If {@code true} (used for certificate rotation), only newly created + * channels are kept so no traffic is routed to a channel using the old certificate, and the + * pool is then refilled asynchronously to its size before the refresh. + * @return if {@code dropUnrefreshedChannels} is {@code false}, whether every channel was + * recreated; otherwise, whether at least one channel was recreated (i.e. every channel left + * in the pool was newly created) + */ + @InternalApi("Visible for testing") + boolean refreshAll(boolean dropUnrefreshedChannels) { + synchronized (entryWriteLock) { + if (isShutdown) { + return false; + } LOG.fine("Refreshing all channels"); - ArrayList newEntries = new ArrayList<>(entries.get()); + // All writes to entries happen under entryWriteLock, so this snapshot is the current pool. + ImmutableList currentEntries = entries.get(); + List keptEntries = new ArrayList<>(currentEntries.size()); + List retiredEntries = new ArrayList<>(currentEntries.size()); + List createdEntries = new ArrayList<>(currentEntries.size()); + boolean allCreated = !currentEntries.isEmpty(); - for (int i = 0; i < newEntries.size(); i++) { - try { - newEntries.set(i, new Entry(channelFactory.createSingleChannel())); - } catch (IOException e) { - LOG.log(Level.WARNING, "Failed to refresh channel, leaving old channel", e); + try { + for (Entry oldEntry : currentEntries) { + try { + Entry newEntry = new Entry(channelFactory.createSingleChannel()); + createdEntries.add(newEntry); + keptEntries.add(newEntry); + retiredEntries.add(oldEntry); + } catch (Exception e) { + allCreated = false; + if (dropUnrefreshedChannels) { + retiredEntries.add(oldEntry); + } else { + keptEntries.add(oldEntry); + } + LOG.log( + Level.WARNING, + dropUnrefreshedChannels + ? "Failed to refresh channel, dropping old channel" + : "Failed to refresh channel, leaving old channel", + e); + } } - } - ImmutableList replacedEntries = entries.getAndSet(ImmutableList.copyOf(newEntries)); + if (createdEntries.isEmpty()) { + return false; + } + + entries.set(ImmutableList.copyOf(keptEntries)); + createdEntries.clear(); // Ownership transferred to pool + + // Shutdown the channels that were cycled out. + retiredEntries.forEach(Entry::requestShutdown); - // Shutdown the channels that were cycled out. - for (Entry e : replacedEntries) { - if (!newEntries.contains(e)) { + // Restore the pool to its pre-refresh size right away, rather than leave a dynamically + // sized + // pool to grow back over several resize() runs while traffic queues on fewer channels. + if (dropUnrefreshedChannels && !allCreated) { + scheduleRefill(currentEntries.size()); + } + return dropUnrefreshedChannels || allCreated; + } finally { + // If an Error aborted before the swap, shut down newly created channels so they don't leak + for (Entry e : createdEntries) { e.requestShutdown(); } } } } + /** + * Schedules a one-shot task that restores the pool to {@code targetSize} channels after a + * certificate rotation refresh dropped channels that failed to refresh. + */ + private void scheduleRefill(int targetSize) { + try { + backgroundExecutorProvider.getExecutor().execute(() -> refillSafely(targetSize)); + } catch (RuntimeException e) { + LOG.log(Level.WARNING, "Failed to schedule channel pool refill", e); + } + } + + private void refillSafely(int targetSize) { + try { + synchronized (entryWriteLock) { + if (isShutdown) { + return; + } + if (entries.get().size() < targetSize) { + expand(targetSize); + } + } + } catch (Exception e) { + LOG.log(Level.WARNING, "Failed to refill channel pool", e); + } + } + + /** + * Returns the current channel pool generation counter. + * + *

    The generation is a monotonically increasing counter incremented each time the pool switches + * every channel to a new mTLS certificate (a reactive refresh after a rotation, or a periodic + * refresh that picks up a rotated certificate). Periodic refreshes that do not change the + * certificate do not increment it. Retry loops ({@code AttemptCallable} and {@code + * ServerStreamingAttemptCallable}) snapshot the generation before starting an RPC attempt and + * compare it after an {@code UNAUTHENTICATED} failure to determine whether the pool rotated to a + * new certificate during or after the attempt. + */ + long getGeneration() { + return generation.get(); + } + /** * Get and retain a Channel Entry. The returned Entry will have its rpc count incremented, * preventing it from getting recycled. @@ -616,17 +860,30 @@ public ClientCall newCall( MethodDescriptor methodDescriptor, CallOptions callOptions) { Entry entry = getRetainedEntry(affinity); - - return new ReleasingClientCall<>(entry.channel.newCall(methodDescriptor, callOptions), entry); + try { + return new ReleasingClientCall<>( + entry.channel.newCall(methodDescriptor, callOptions), entry); + } catch (Throwable t) { + entry.release(); + throw t; + } } } - /** ClientCall wrapper that makes sure to decrement the outstanding RPC count on completion. */ + /** + * ClientCall wrapper that makes sure to decrement the outstanding RPC count on completion. + * + *

    Contract: Exactly one call to {@link #start(Listener, Metadata)} is required to balance + * reference counts. Early cancellation before {@code start()} is recorded and safely decrements + * the reference count when {@code start()} is subsequently invoked. + */ static class ReleasingClientCall extends SimpleForwardingClientCall { - private @Nullable CancellationException cancellationException; + private final Object callLock = new Object(); + private volatile @Nullable CancellationException cancellationException; final Entry entry; private final AtomicBoolean wasClosed = new AtomicBoolean(); private final AtomicBoolean wasReleased = new AtomicBoolean(); + private final AtomicBoolean wasStarted = new AtomicBoolean(); public ReleasingClientCall(ClientCall delegate, Entry entry) { super(delegate); @@ -635,51 +892,81 @@ public ReleasingClientCall(ClientCall delegate, Entry entry) { @Override public void start(Listener responseListener, Metadata headers) { - if (cancellationException != null) { - throw new IllegalStateException("Call is already cancelled", cancellationException); - } - try { - super.start( - new SimpleForwardingClientCallListener(responseListener) { - @Override - public void onClose(Status status, Metadata trailers) { - if (!wasClosed.compareAndSet(false, true)) { - LOG.log( - Level.WARNING, - "Call is being closed more than once. Please make sure that onClose() is not being manually called."); - return; - } - try { - super.onClose(status, trailers); - } finally { - if (wasReleased.compareAndSet(false, true)) { - entry.release(); - } else { + synchronized (callLock) { + if (!wasStarted.compareAndSet(false, true)) { + throw new IllegalStateException("Call is already started"); + } + if (cancellationException != null) { + if (wasReleased.compareAndSet(false, true)) { + entry.release(); + } + throw new IllegalStateException("Call is already cancelled", cancellationException); + } + try { + super.start( + new SimpleForwardingClientCallListener(responseListener) { + @Override + public void onClose(Status status, Metadata trailers) { + if (!wasClosed.compareAndSet(false, true)) { LOG.log( Level.WARNING, - "Entry was released before the call is closed. This may be due to an exception on start of the call."); + "Call is being closed more than once. Please make sure that onClose() is" + + " not being manually called."); + return; + } + try { + super.onClose(status, trailers); + } finally { + if (wasReleased.compareAndSet(false, true)) { + entry.release(); + } else { + LOG.log( + Level.WARNING, + "Entry was released before the call is closed. This may be due to an" + + " exception on start of the call."); + } } } - } - }, - headers); - } catch (Exception e) { - // In case start failed, make sure to release - if (wasReleased.compareAndSet(false, true)) { - entry.release(); - } else { - LOG.log( - Level.WARNING, - "The entry is already released. This indicates that onClose() has already been called previously"); + }, + headers); + } catch (Throwable t) { + // In case start failed, make sure to release + if (wasReleased.compareAndSet(false, true)) { + entry.release(); + } else { + LOG.log( + Level.WARNING, + "The entry is already released. This indicates that onClose() has already been" + + " called previously"); + } + throw t; } - throw e; } } @Override public void cancel(@Nullable String message, @Nullable Throwable cause) { - this.cancellationException = new CancellationException(message); - super.cancel(message, cause); + boolean releaseImmediately = false; + try { + synchronized (callLock) { + this.cancellationException = new CancellationException(message); + if (!wasStarted.get()) { + releaseImmediately = true; + } + if (delegate() != null) { + super.cancel(message, cause); + } + } + } catch (Throwable t) { + if (!wasStarted.get()) { + releaseImmediately = true; + } + throw t; + } finally { + if (releaseImmediately && wasReleased.compareAndSet(false, true)) { + entry.release(); + } + } } } } diff --git a/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/GrpcCallContext.java b/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/GrpcCallContext.java index 2337ca2f6f07..8d4080247ec4 100644 --- a/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/GrpcCallContext.java +++ b/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/GrpcCallContext.java @@ -98,6 +98,7 @@ public final class GrpcCallContext implements ApiCallContext { private final ApiCallContextOptions options; private final EndpointContext endpointContext; private final boolean isDirectPath; + @Nullable private final TransportChannel transportChannel; /** Returns an empty instance with a null channel and default {@link CallOptions}. */ public static GrpcCallContext createDefault() { @@ -114,7 +115,8 @@ public static GrpcCallContext createDefault() { null, null, null, - false); + false, + null); } /** Returns an instance with the given channel and {@link CallOptions}. */ @@ -132,7 +134,8 @@ public static GrpcCallContext of(Channel channel, CallOptions callOptions) { null, null, null, - false); + false, + null); } private GrpcCallContext( @@ -148,7 +151,8 @@ private GrpcCallContext( @Nullable RetrySettings retrySettings, @Nullable Set retryableCodes, @Nullable EndpointContext endpointContext, - boolean isDirectPath) { + boolean isDirectPath, + @Nullable TransportChannel transportChannel) { this.channel = channel; this.credentials = credentials; Preconditions.checkNotNull(callOptions); @@ -168,6 +172,7 @@ private GrpcCallContext( this.endpointContext = endpointContext == null ? EndpointContext.getDefaultInstance() : endpointContext; this.isDirectPath = isDirectPath; + this.transportChannel = transportChannel; } /** @@ -209,7 +214,13 @@ public GrpcCallContext withCredentials(Credentials newCredentials) { retrySettings, retryableCodes, endpointContext, - isDirectPath); + isDirectPath, + transportChannel); + } + + @Override + public TransportChannel getTransportChannel() { + return transportChannel; } @Override @@ -233,7 +244,8 @@ public GrpcCallContext withTransportChannel(TransportChannel inputChannel) { retrySettings, retryableCodes, endpointContext, - transportChannel.isDirectPath()); + transportChannel.isDirectPath(), + inputChannel); } @Override @@ -252,7 +264,8 @@ public GrpcCallContext withEndpointContext(EndpointContext endpointContext) { retrySettings, retryableCodes, endpointContext, - isDirectPath); + isDirectPath, + transportChannel); } /** This method is obsolete. Use {@link #withTimeoutDuration(java.time.Duration)} instead. */ @@ -287,7 +300,8 @@ public GrpcCallContext withTimeoutDuration(java.time.@Nullable Duration timeout) retrySettings, retryableCodes, endpointContext, - isDirectPath); + isDirectPath, + transportChannel); } @Override @@ -333,7 +347,8 @@ public GrpcCallContext withStreamWaitTimeoutDuration( retrySettings, retryableCodes, endpointContext, - isDirectPath); + isDirectPath, + transportChannel); } /** @@ -368,7 +383,8 @@ public GrpcCallContext withStreamIdleTimeoutDuration( retrySettings, retryableCodes, endpointContext, - isDirectPath); + isDirectPath, + transportChannel); } @BetaApi("The surface for channel affinity is not stable yet and may change in the future.") @@ -386,7 +402,8 @@ public GrpcCallContext withChannelAffinity(@Nullable Integer affinity) { retrySettings, retryableCodes, endpointContext, - isDirectPath); + isDirectPath, + transportChannel); } @BetaApi("The surface for extra headers is not stable yet and may change in the future.") @@ -408,7 +425,8 @@ public GrpcCallContext withExtraHeaders(Map> extraHeaders) retrySettings, retryableCodes, endpointContext, - isDirectPath); + isDirectPath, + transportChannel); } @Override @@ -431,7 +449,8 @@ public GrpcCallContext withRetrySettings(RetrySettings retrySettings) { retrySettings, retryableCodes, endpointContext, - isDirectPath); + isDirectPath, + transportChannel); } @Override @@ -454,7 +473,8 @@ public GrpcCallContext withRetryableCodes(Set retryableCodes) { retrySettings, retryableCodes, endpointContext, - isDirectPath); + isDirectPath, + transportChannel); } @Override @@ -541,6 +561,12 @@ public ApiCallContext merge(ApiCallContext inputCallContext) { newCallOptions = newCallOptions.withOption(TRACER_KEY, newTracer); } + TransportChannel newTransportChannel = grpcCallContext.transportChannel; + if (newTransportChannel == null + && (grpcCallContext.channel == null || grpcCallContext.channel.equals(channel))) { + newTransportChannel = transportChannel; + } + // The EndpointContext is not updated as there should be no reason for a user // to update this. return new GrpcCallContext( @@ -556,7 +582,8 @@ public ApiCallContext merge(ApiCallContext inputCallContext) { newRetrySettings, newRetryableCodes, endpointContext, - newIsDirectPath); + newIsDirectPath, + newTransportChannel); } /** The {@link Channel} set on this context. */ @@ -634,7 +661,8 @@ public GrpcCallContext withChannel(@Nullable Channel newChannel) { retrySettings, retryableCodes, endpointContext, - isDirectPath); + isDirectPath, + (newChannel != null && newChannel.equals(channel)) ? transportChannel : null); } /** Returns a new instance with the call options set to the given call options. */ @@ -652,7 +680,8 @@ public GrpcCallContext withCallOptions(CallOptions newCallOptions) { retrySettings, retryableCodes, endpointContext, - isDirectPath); + isDirectPath, + transportChannel); } public GrpcCallContext withRequestParamsDynamicHeaderOption(String requestParams) { @@ -696,7 +725,8 @@ public GrpcCallContext withOption(Key key, T value) { retrySettings, retryableCodes, endpointContext, - isDirectPath); + isDirectPath, + transportChannel); } /** {@inheritDoc} */ @@ -757,7 +787,8 @@ public int hashCode() { options, retrySettings, retryableCodes, - endpointContext); + endpointContext, + transportChannel); } @Override @@ -781,7 +812,8 @@ public boolean equals(@Nullable Object o) { && Objects.equals(options, that.options) && Objects.equals(retrySettings, that.retrySettings) && Objects.equals(retryableCodes, that.retryableCodes) - && Objects.equals(endpointContext, that.endpointContext); + && Objects.equals(endpointContext, that.endpointContext) + && Objects.equals(transportChannel, that.transportChannel); } Metadata getMetadata() { diff --git a/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/GrpcTransportChannel.java b/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/GrpcTransportChannel.java index e0a520facb17..63180a8149f6 100644 --- a/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/GrpcTransportChannel.java +++ b/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/GrpcTransportChannel.java @@ -68,6 +68,32 @@ public Channel getChannel() { return getManagedChannel(); } + @Override + public void refresh() { + Channel channel = getChannel(); + if (channel instanceof ChannelPool) { + ((ChannelPool) channel).refresh(); + } + } + + @Override + public boolean shouldRefresh() { + Channel channel = getChannel(); + if (channel instanceof ChannelPool) { + return ((ChannelPool) channel).shouldRefresh(); + } + return false; + } + + @Override + public long getGeneration() { + Channel channel = getChannel(); + if (channel instanceof ChannelPool) { + return ((ChannelPool) channel).getGeneration(); + } + return 0; + } + @Override public void shutdown() { getManagedChannel().shutdown(); diff --git a/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java b/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java index f300ac611da8..91299357da10 100644 --- a/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java +++ b/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java @@ -46,7 +46,6 @@ import com.google.auth.ApiKeyCredentials; import com.google.auth.Credentials; import com.google.auth.mtls.CertificateSourceUnavailableException; -import com.google.auth.mtls.DefaultMtlsProviderFactory; import com.google.auth.mtls.MtlsProvider; import com.google.auth.oauth2.ComputeEngineCredentials; import com.google.auth.oauth2.SecureSessionAgent; @@ -400,12 +399,21 @@ public TransportChannel getTransportChannel() throws IOException { } private TransportChannel createChannel() throws IOException { + // Only track the workload certificate for rotation when the pool's channels actually present + // it, mirroring the credential selection in createSingleChannel(): DirectPath channels use + // GoogleDefaultChannelCredentials (ALTS) rather than the client certificate, and without an + // mtlsProvider there is no client certificate KeyStore (S2A or plain TLS is used instead). + String workloadCertPath = + !this.canUseDirectPath() && mtlsProvider != null + ? certificateBasedAccess.getWorkloadCertPath() + : null; return GrpcTransportChannel.newBuilder() .setManagedChannel( ChannelPool.create( channelPoolSettings, InstantiatingGrpcChannelProvider.this::createSingleChannel, - backgroundExecutor)) + backgroundExecutor, + workloadCertPath)) .setDirectPath(this.canUseDirectPath()) .build(); } @@ -464,8 +472,9 @@ private void logDirectPathMisconfig() { level, "Env var " + DIRECT_PATH_ENV_ENABLE_XDS - + " was found and set to TRUE, but DirectPath was not enabled for this client. If this is intended for " - + "this client, please note that this is a misconfiguration and set the attemptDirectPath option as well."); + + " was found and set to TRUE, but DirectPath was not enabled for this client. If" + + " this is intended for this client, please note that this is a misconfiguration" + + " and set the attemptDirectPath option as well."); } // Case 2: Direct Path xDS was enabled via Builder. Direct Path Traffic Director must be set // (enabled with `setAttemptDirectPath(true)`) along with xDS. @@ -473,7 +482,9 @@ private void logDirectPathMisconfig() { else if (isDirectPathXdsEnabledViaBuilderOption()) { LOG.log( level, - "DirectPath is misconfigured. The DirectPath XDS option was set, but the attemptDirectPath option was not. Please set both the attemptDirectPath and attemptDirectPathXds options."); + "DirectPath is misconfigured. The DirectPath XDS option was set, but the" + + " attemptDirectPath option was not. Please set both the attemptDirectPath and" + + " attemptDirectPathXds options."); } } else { // Case 3: credential is not correctly set @@ -664,7 +675,8 @@ ChannelCredentials createS2ASecuredChannelCredentials() { // Fallback to plaintext connection to S2A. LOG.log( Level.INFO, - "Cannot establish an mTLS connection to S2A because autoconfig endpoint did not return a mtls address to reach S2A."); + "Cannot establish an mTLS connection to S2A because autoconfig endpoint did not" + + " return a mtls address to reach S2A."); s2aChannelCredentials = createPlaintextToS2AChannelCredentials(plaintextAddress); return s2aChannelCredentials; } @@ -683,7 +695,9 @@ ChannelCredentials createS2ASecuredChannelCredentials() { // Fallback to plaintext-to-S2A connection on error. LOG.log( Level.WARNING, - "Cannot establish an mTLS connection to S2A due to error creating MTLS to MDS TlsChannelCredentials credentials, falling back to plaintext connection to S2A: " + "Cannot establish an mTLS connection to S2A due to error creating MTLS to MDS" + + " TlsChannelCredentials credentials, falling back to plaintext connection to" + + " S2A: " + ignore.getMessage()); s2aChannelCredentials = createPlaintextToS2AChannelCredentials(plaintextAddress); return s2aChannelCredentials; @@ -753,6 +767,8 @@ public ManagedChannelBuilder createChannelBuilder() throws IOException { if (channelCredentials != null) { // Create the channel using channel credentials created via DCA. builder = Grpc.newChannelBuilder(endpoint, channelCredentials); + } else if (mtlsProvider != null && certificateBasedAccess.useMtlsClientCertificate()) { + throw new IOException("Failed to initialize mTLS channel credentials"); } else { // Could not create channel credentials via DCA. In accordance with // https://google.aip.dev/auth/4115, if credentials not available through @@ -1391,17 +1407,23 @@ public InstantiatingGrpcChannelProvider build() { if (mtlsProvider == null) { // Attempt to create default MtlsProvider from environment. try { - mtlsProvider = DefaultMtlsProviderFactory.create(); + mtlsProvider = certificateBasedAccess.createDefaultMtlsProvider(); } catch (CertificateSourceUnavailableException e) { // This is okay. Leave mtlsProvider as null so that we will not auto-upgrade // to mTLS endpoints. See https://google.aip.dev/auth/4114. } catch (IOException e) { - LOG.log( - Level.WARNING, - "DefaultMtlsProviderFactory encountered unexpected IOException: " + e.getMessage()); - LOG.log( - Level.WARNING, - "mTLS configuration was detected on the device, but mTLS failed to initialize. Falling back to non-mTLS channel."); + mtlsProvider = + new MtlsProvider() { + @Override + public boolean isAvailable() throws IOException { + throw e; + } + + @Override + public KeyStore getKeyStore() throws IOException { + throw e; + } + }; } } } diff --git a/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java b/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java index 5bfdc7754759..ed658c2f45d8 100644 --- a/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java +++ b/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/ChannelPoolTest.java @@ -47,6 +47,7 @@ import com.google.api.gax.rpc.StreamController; import com.google.api.gax.rpc.UnaryCallSettings; import com.google.api.gax.rpc.UnaryCallable; +import com.google.api.gax.rpc.mtls.CertificateRotationTracker; import com.google.api.gax.util.FakeLogHandler; import com.google.auth.Credentials; import com.google.common.collect.ImmutableList; @@ -61,16 +62,27 @@ import io.grpc.Status; import io.grpc.stub.ClientCalls; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.concurrent.CancellationException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -81,13 +93,18 @@ class ChannelPoolTest { private static final int DEFAULT_AWAIT_TERMINATION_SEC = 10; private ChannelPool pool; + private Path tempCert; @AfterEach - void cleanup() throws InterruptedException { + void cleanup() throws InterruptedException, IOException { if (pool != null) { pool.shutdown(); pool.awaitTermination(DEFAULT_AWAIT_TERMINATION_SEC, TimeUnit.SECONDS); } + if (tempCert != null) { + Files.deleteIfExists(tempCert); + tempCert = null; + } } @Test @@ -101,6 +118,7 @@ void testAuthority() throws IOException { ChannelPool.create( ChannelPoolSettings.staticallySized(2), new FakeChannelFactory(Arrays.asList(sub1, sub2)), + null, null); assertThat(pool.authority()).isEqualTo("myAuth"); } @@ -117,6 +135,7 @@ void testRoundRobin() throws IOException { ChannelPool.create( ChannelPoolSettings.staticallySized(channels.size()), new FakeChannelFactory(channels), + null, null); verifyTargetChannel(pool, channels, sub1); @@ -195,6 +214,7 @@ void ensureEvenDistribution() throws InterruptedException, IOException { ChannelPool.create( ChannelPoolSettings.staticallySized(numChannels), new FakeChannelFactory(Arrays.asList(channels)), + null, null); int numThreads = 20; @@ -233,6 +253,7 @@ void channelPrimerShouldCallPoolConstruction() throws IOException { .setPreemptiveRefreshEnabled(true) .build(), new FakeChannelFactory(Arrays.asList(channel1, channel2), mockChannelPrimer), + null, null); Mockito.verify(mockChannelPrimer, Mockito.times(2)) .primeChannel(Mockito.any(ManagedChannel.class)); @@ -273,7 +294,8 @@ void channelPrimerIsCalledPeriodically() throws IOException { .setPreemptiveRefreshEnabled(true) .build(), channelFactory, - provider); + provider, + null); // 1 call during the creation Mockito.verify(mockChannelPrimer, Mockito.times(1)) .primeChannel(Mockito.any(ManagedChannel.class)); @@ -297,7 +319,7 @@ void callShouldCompleteAfterCreation() throws IOException { ManagedChannel replacementChannel = mock(ManagedChannel.class); FakeChannelFactory channelFactory = new FakeChannelFactory(ImmutableList.of(underlyingChannel, replacementChannel)); - pool = ChannelPool.create(ChannelPoolSettings.staticallySized(1), channelFactory, null); + pool = ChannelPool.create(ChannelPoolSettings.staticallySized(1), channelFactory, null, null); // create a mock call when new call comes to the underlying channel MockClientCall mockClientCall = new MockClientCall<>(1, Status.OK); @@ -322,7 +344,7 @@ void callShouldCompleteAfterCreation() throws IOException { ClientCall call = pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); - pool.refresh(); + pool.refreshAll(); // shutdown is not called because there is still an outstanding call, even if it hasn't started Mockito.verify(underlyingChannel, Mockito.after(200).never()).shutdown(); @@ -346,7 +368,7 @@ void callShouldCompleteAfterStarted() throws IOException { FakeChannelFactory channelFactory = new FakeChannelFactory(ImmutableList.of(underlyingChannel, replacementChannel)); - pool = ChannelPool.create(ChannelPoolSettings.staticallySized(1), channelFactory, null); + pool = ChannelPool.create(ChannelPoolSettings.staticallySized(1), channelFactory, null, null); // create a mock call when new call comes to the underlying channel MockClientCall mockClientCall = new MockClientCall<>(1, Status.OK); @@ -373,7 +395,7 @@ void callShouldCompleteAfterStarted() throws IOException { // start clientCall call.start(listener, new Metadata()); - pool.refresh(); + pool.refreshAll(); // shutdown is not called because there is still an outstanding call Mockito.verify(underlyingChannel, Mockito.after(200).never()).shutdown(); @@ -391,7 +413,7 @@ void channelShouldShutdown() throws IOException { FakeChannelFactory channelFactory = new FakeChannelFactory(ImmutableList.of(underlyingChannel, replacementChannel)); - pool = ChannelPool.create(ChannelPoolSettings.staticallySized(1), channelFactory, null); + pool = ChannelPool.create(ChannelPoolSettings.staticallySized(1), channelFactory, null, null); // create a mock call when new call comes to the underlying channel MockClientCall mockClientCall = new MockClientCall<>(1, Status.OK); @@ -422,11 +444,1009 @@ void channelShouldShutdown() throws IOException { call.sendMessage("message"); // shutdown is not called because it has not been shutdown yet Mockito.verify(underlyingChannel, Mockito.after(200).never()).shutdown(); - pool.refresh(); + pool.refreshAll(); // shutdown is called because the outstanding call has completed Mockito.verify(underlyingChannel, Mockito.atLeastOnce()).shutdown(); } + @Test + void testCancelBeforeStartReleasesChannelEntry() throws IOException { + ManagedChannel underlyingChannel = mock(ManagedChannel.class); + ManagedChannel replacementChannel = mock(ManagedChannel.class); + FakeChannelFactory channelFactory = + new FakeChannelFactory(ImmutableList.of(underlyingChannel, replacementChannel)); + pool = ChannelPool.create(ChannelPoolSettings.staticallySized(1), channelFactory, null, null); + + ClientCall call = + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + + pool.refreshAll(); + Mockito.verify(underlyingChannel, Mockito.never()).shutdown(); + + call.cancel("Cancelled early", null); + Mockito.verify(underlyingChannel, Mockito.times(1)).shutdown(); + } + + @Test + void channelReactiveMTlsRefresh_swapsChannelsOnlyWhenCertChanges() + throws IOException, InterruptedException { + ManagedChannel underlyingChannel1 = Mockito.mock(ManagedChannel.class); + ManagedChannel underlyingChannel2 = Mockito.mock(ManagedChannel.class); + + FakeChannelFactory channelFactory = + new FakeChannelFactory(ImmutableList.of(underlyingChannel1, underlyingChannel2)); + + // Create a temp file to act as the cert + tempCert = Files.createTempFile("cert", ".pem"); + + Path clientCert = Paths.get("src", "test", "resources", "client_cert.pem"); + Files.copy(clientCert, tempCert, StandardCopyOption.REPLACE_EXISTING); + + ChannelPoolSettings channelPoolSettings = + ChannelPoolSettings.builder().setInitialChannelCount(1).build(); + + pool = ChannelPool.create(channelPoolSettings, channelFactory, null, tempCert.toString()); + + // Initially uses channel1 + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + Mockito.verify(underlyingChannel1, Mockito.times(1)) + .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); + + // Try a reactive refresh *without* changing the cert content (should no-op) + pool.refresh(); + + // Verify it's STILL channel1 + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + Mockito.verify(underlyingChannel1, Mockito.times(2)) + .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); + + // The ChannelPool caches fingerprints for 1000ms, wait for it to expire + pool.invalidateDiskFingerprintCache(); + + Path rootCert = Paths.get("src", "test", "resources", "root_cert.pem"); + Files.copy(rootCert, tempCert, StandardCopyOption.REPLACE_EXISTING); + + // Try a reactive refresh *with* a changed cert content (should swap channels) + pool.refresh(); + + // Verify it is NOW channel2 + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + Mockito.verify(underlyingChannel2, Mockito.times(1)) + .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); + } + + @Test + void channelReactiveMTlsRefresh_failedCreationDoesNotMutateFingerprintAndAllowsRetry() + throws IOException { + ManagedChannel channel1 = Mockito.mock(ManagedChannel.class); + ManagedChannel channel2 = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = + Mockito.mock(ChannelFactory.class, Mockito.withSettings().withoutAnnotations()); + + // Initial creation returns channel1, refresh attempt 1 throws IOException, refresh attempt 2 + // returns channel2 + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(channel1) + .thenThrow(new IOException("Transient channel creation error")) + .thenReturn(channel2); + + tempCert = Files.createTempFile("cert", ".pem"); + Path clientCert = Paths.get("src", "test", "resources", "client_cert.pem"); + Files.copy(clientCert, tempCert, StandardCopyOption.REPLACE_EXISTING); + + ChannelPoolSettings channelPoolSettings = + ChannelPoolSettings.builder().setInitialChannelCount(1).build(); + + pool = ChannelPool.create(channelPoolSettings, channelFactory, null, tempCert.toString()); + + // Initially uses channel1 + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + Mockito.verify(channel1, Mockito.times(1)) + .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); + + // Rotate cert on disk + pool.invalidateDiskFingerprintCache(); + Path rootCert = Paths.get("src", "test", "resources", "root_cert.pem"); + Files.copy(rootCert, tempCert, StandardCopyOption.REPLACE_EXISTING); + + // Refresh attempt 1: createSingleChannel throws IOException. + // Refresh should fail to replace channel and MUST NOT record the new cert fingerprint as + // active. + pool.refresh(); + + // Verify still channel1 + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + Mockito.verify(channel1, Mockito.times(2)) + .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); + + // Refresh attempt 2: with the same cert file on disk (cache expired), channelFactory now + // succeeds. + // If the fingerprint had been mutated on the failed attempt, this call would be skipped as a + // duplicate! + pool.invalidateDiskFingerprintCache(); + pool.refresh(); + + // Verify it has now swapped to channel2! + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + Mockito.verify(channel2, Mockito.times(1)) + .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); + } + + private void writeCert(String resourceName) throws IOException { + if (tempCert == null) { + tempCert = Files.createTempFile("cert", ".pem"); + } + Files.copy( + Paths.get("src", "test", "resources", resourceName), + tempCert, + StandardCopyOption.REPLACE_EXISTING); + } + + /** Creates an mTLS pool backed by {@code executor} and then rotates the certificate on disk. */ + private ChannelPool createMtlsPoolAndRotateCert( + ChannelPoolSettings settings, + ChannelFactory channelFactory, + ScheduledExecutorService executor) + throws IOException { + writeCert("client_cert.pem"); + pool = + new ChannelPool( + settings, channelFactory, FixedExecutorProvider.create(executor), tempCert.toString()); + pool.invalidateDiskFingerprintCache(); + writeCert("root_cert.pem"); + assertThat(pool.shouldRefresh()).isTrue(); + return pool; + } + + private static ScheduledExecutorService mockExecutor() { + return Mockito.mock( + ScheduledExecutorService.class, Mockito.withSettings().withoutAnnotations()); + } + + private static ChannelFactory mockChannelFactory() { + return Mockito.mock(ChannelFactory.class, Mockito.withSettings().withoutAnnotations()); + } + + @Test + void channelReactiveMTlsRefresh_partialFailureInStaticPool_dropsUnrefreshedChannelsAndRefills() + throws IOException { + ScheduledExecutorService executor = mockExecutor(); + ManagedChannel initial1 = Mockito.mock(ManagedChannel.class); + ManagedChannel initial2 = Mockito.mock(ManagedChannel.class); + ManagedChannel rotated1 = Mockito.mock(ManagedChannel.class); + ManagedChannel refilled = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(initial1, initial2) + .thenReturn(rotated1) + .thenThrow(new IOException("Transient failure on second sub-channel")) + .thenReturn(refilled); + + createMtlsPoolAndRotateCert(ChannelPoolSettings.staticallySized(2), channelFactory, executor); + long genBefore = pool.getGeneration(); + + pool.refresh(); + + // Only the newly created channel is kept; both channels on the old certificate are retired. + assertThat(pool.entries.get()).hasSize(1); + Mockito.verify(initial1).shutdown(); + Mockito.verify(initial2).shutdown(); + Mockito.verify(rotated1, Mockito.never()).shutdown(); + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + Mockito.verify(rotated1) + .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); + assertThat(pool.getGeneration()).isEqualTo(genBefore + 1); + // Every channel left in the pool uses the new certificate, so it is recorded as active. + pool.invalidateDiskFingerprintCache(); + assertThat(pool.shouldRefresh()).isFalse(); + + // A one-time refill restores the configured size. + ArgumentCaptor refillTask = ArgumentCaptor.forClass(Runnable.class); + Mockito.verify(executor).execute(refillTask.capture()); + refillTask.getValue().run(); + assertThat(pool.entries.get()).hasSize(2); + Mockito.verify(channelFactory, Mockito.times(5)).createSingleChannel(); + + // Running the refill again on a full pool creates no channels. + refillTask.getValue().run(); + assertThat(pool.entries.get()).hasSize(2); + Mockito.verify(channelFactory, Mockito.times(5)).createSingleChannel(); + Mockito.verify(refilled, Mockito.never()).shutdown(); + Mockito.verify(executor, Mockito.times(1)).execute(Mockito.any(Runnable.class)); + } + + @Test + void channelReactiveMTlsRefresh_fullSuccess_doesNotScheduleRefill() throws IOException { + ScheduledExecutorService executor = mockExecutor(); + ManagedChannel initial1 = Mockito.mock(ManagedChannel.class); + ManagedChannel initial2 = Mockito.mock(ManagedChannel.class); + ManagedChannel rotated1 = Mockito.mock(ManagedChannel.class); + ManagedChannel rotated2 = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(initial1, initial2, rotated1, rotated2); + + createMtlsPoolAndRotateCert(ChannelPoolSettings.staticallySized(2), channelFactory, executor); + + pool.refresh(); + + assertThat(pool.entries.get()).hasSize(2); + Mockito.verify(initial1).shutdown(); + Mockito.verify(initial2).shutdown(); + pool.invalidateDiskFingerprintCache(); + assertThat(pool.shouldRefresh()).isFalse(); + Mockito.verify(executor, Mockito.never()).execute(Mockito.any(Runnable.class)); + } + + @Test + void refreshAll_partialFailureWithoutRotation_keepsOldChannelAndDoesNotScheduleRefill() + throws IOException { + // Non-rotation refreshes (e.g. the preemptive refresh) keep per-slot fallback even for mTLS. + ScheduledExecutorService executor = mockExecutor(); + ManagedChannel initial1 = Mockito.mock(ManagedChannel.class); + ManagedChannel initial2 = Mockito.mock(ManagedChannel.class); + ManagedChannel refreshed1 = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(initial1, initial2) + .thenReturn(refreshed1) + .thenThrow(new IOException("Transient failure on second sub-channel")); + + writeCert("client_cert.pem"); + pool = + new ChannelPool( + ChannelPoolSettings.staticallySized(2), + channelFactory, + FixedExecutorProvider.create(executor), + tempCert.toString()); + long genBefore = pool.getGeneration(); + List before = pool.entries.get(); + + assertThat(pool.refreshAll()).isFalse(); + + assertThat(pool.entries.get()).hasSize(2); + // Each channel keeps its slot, so affinity-to-index mapping is unchanged. + assertThat(pool.entries.get().get(0)).isNotSameInstanceAs(before.get(0)); + assertThat(pool.entries.get().get(1)).isSameInstanceAs(before.get(1)); + Mockito.verify(initial1).shutdown(); + Mockito.verify(initial2, Mockito.never()).shutdown(); + assertThat(pool.getGeneration()).isEqualTo(genBefore); + Mockito.verify(executor, Mockito.never()).execute(Mockito.any(Runnable.class)); + } + + @Test + void channelReactiveMTlsRefresh_allChannelsFail_leavesPoolUnchangedAndDoesNotRefill() + throws IOException { + ScheduledExecutorService executor = mockExecutor(); + ManagedChannel initial1 = Mockito.mock(ManagedChannel.class); + ManagedChannel initial2 = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(initial1, initial2) + .thenThrow(new IOException("Failure on first sub-channel")) + .thenThrow(new IOException("Failure on second sub-channel")); + + createMtlsPoolAndRotateCert(ChannelPoolSettings.staticallySized(2), channelFactory, executor); + long genBefore = pool.getGeneration(); + List before = pool.entries.get(); + + pool.refresh(); + + // Nothing could be rebuilt, so the pool keeps its old channels and the rotation stays pending. + assertThat(pool.entries.get()).containsExactlyElementsIn(before).inOrder(); + Mockito.verify(initial1, Mockito.never()).shutdown(); + Mockito.verify(initial2, Mockito.never()).shutdown(); + assertThat(pool.getGeneration()).isEqualTo(genBefore); + pool.invalidateDiskFingerprintCache(); + assertThat(pool.shouldRefresh()).isTrue(); + Mockito.verify(executor, Mockito.never()).execute(Mockito.any(Runnable.class)); + } + + @Test + void channelReactiveMTlsRefresh_partialFailureInDynamicPool_refillsToPreRefreshSize() + throws IOException { + ScheduledExecutorService executor = mockExecutor(); + ManagedChannel initial1 = Mockito.mock(ManagedChannel.class); + ManagedChannel initial2 = Mockito.mock(ManagedChannel.class); + ManagedChannel rotated1 = Mockito.mock(ManagedChannel.class); + ManagedChannel refilled = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(initial1, initial2) + .thenReturn(rotated1) + .thenThrow(new IOException("Transient failure on second sub-channel")) + .thenReturn(refilled); + + createMtlsPoolAndRotateCert( + ChannelPoolSettings.builder() + .setInitialChannelCount(2) + .setMinChannelCount(2) + .setMaxChannelCount(4) + .setMinRpcsPerChannel(1) + .setMaxRpcsPerChannel(2) + .build(), + channelFactory, + executor); + + pool.refresh(); + + assertThat(pool.entries.get()).hasSize(1); + Mockito.verify(initial1).shutdown(); + Mockito.verify(initial2).shutdown(); + pool.invalidateDiskFingerprintCache(); + assertThat(pool.shouldRefresh()).isFalse(); + + // Dynamic pools are refilled right away too, rather than waiting for resize(). + ArgumentCaptor refillTask = ArgumentCaptor.forClass(Runnable.class); + Mockito.verify(executor).execute(refillTask.capture()); + refillTask.getValue().run(); + assertThat(pool.entries.get()).hasSize(2); + Mockito.verify(refilled, Mockito.never()).shutdown(); + } + + @Test + void channelReactiveMTlsRefresh_partialFailureAfterResize_refillsToPreRefreshSize() + throws IOException { + ScheduledExecutorService executor = mockExecutor(); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn( + // initial pool of 4 + Mockito.mock(ManagedChannel.class), + Mockito.mock(ManagedChannel.class), + Mockito.mock(ManagedChannel.class), + Mockito.mock(ManagedChannel.class), + // rotation refresh: 1 of the 2 remaining channels is recreated + Mockito.mock(ManagedChannel.class)) + .thenThrow(new IOException("Transient failure on second sub-channel")) + .thenReturn(Mockito.mock(ManagedChannel.class)); + + createMtlsPoolAndRotateCert( + ChannelPoolSettings.builder() + .setInitialChannelCount(4) + .setMinChannelCount(2) + .setMaxChannelCount(6) + .setMinRpcsPerChannel(1) + .setMaxRpcsPerChannel(2) + .build(), + channelFactory, + executor); + // With no load, resize() shrinks the pool below its initial channel count. + pool.resize(); + assertThat(pool.entries.get()).hasSize(2); + + pool.refresh(); + assertThat(pool.entries.get()).hasSize(1); + + // The refill targets the size before the refresh, not the initial channel count. + ArgumentCaptor refillTask = ArgumentCaptor.forClass(Runnable.class); + Mockito.verify(executor).execute(refillTask.capture()); + refillTask.getValue().run(); + assertThat(pool.entries.get()).hasSize(2); + Mockito.verify(channelFactory, Mockito.times(7)).createSingleChannel(); + } + + @Test + void refill_afterShutdown_createsNoChannels() throws IOException { + ScheduledExecutorService executor = mockExecutor(); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn( + Mockito.mock(ManagedChannel.class), + Mockito.mock(ManagedChannel.class), + Mockito.mock(ManagedChannel.class)) + .thenThrow(new IOException("Transient failure on second sub-channel")) + .thenReturn(Mockito.mock(ManagedChannel.class)); + + createMtlsPoolAndRotateCert(ChannelPoolSettings.staticallySized(2), channelFactory, executor); + pool.refresh(); + ArgumentCaptor refillTask = ArgumentCaptor.forClass(Runnable.class); + Mockito.verify(executor).execute(refillTask.capture()); + + pool.shutdown(); + refillTask.getValue().run(); + + assertThat(pool.entries.get()).hasSize(1); + Mockito.verify(channelFactory, Mockito.times(4)).createSingleChannel(); + } + + @Test + void resize_afterShutdown_createsNoChannels() throws IOException { + ScheduledExecutorService executor = mockExecutor(); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn( + Mockito.mock(ManagedChannel.class), + Mockito.mock(ManagedChannel.class), + Mockito.mock(ManagedChannel.class)) + .thenThrow(new IOException("Transient failure on second sub-channel")) + .thenReturn(Mockito.mock(ManagedChannel.class)); + createMtlsPoolAndRotateCert( + ChannelPoolSettings.builder() + .setInitialChannelCount(2) + .setMinChannelCount(2) + .setMaxChannelCount(4) + .setMinRpcsPerChannel(1) + .setMaxRpcsPerChannel(2) + .build(), + channelFactory, + executor); + ArgumentCaptor resizeTask = ArgumentCaptor.forClass(Runnable.class); + Mockito.verify(executor) + .scheduleAtFixedRate( + resizeTask.capture(), Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + // Drop the pool below its minimum size, so that resize() would expand it. + pool.refresh(); + assertThat(pool.entries.get()).hasSize(1); + + // A resize run that was already waiting for the lock when the pool was shut down. + pool.shutdown(); + resizeTask.getValue().run(); + + assertThat(pool.entries.get()).hasSize(1); + Mockito.verify(channelFactory, Mockito.times(4)).createSingleChannel(); + } + + @Test + void refill_channelCreationFailure_isLoggedAndDoesNotThrow() throws IOException { + ScheduledExecutorService executor = mockExecutor(); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn( + Mockito.mock(ManagedChannel.class), + Mockito.mock(ManagedChannel.class), + Mockito.mock(ManagedChannel.class)) + .thenThrow(new IOException("Transient failure on second sub-channel")) + .thenThrow(new IOException("Checked failure during refill")) + .thenThrow(new RuntimeException("Unchecked failure during refill")) + .thenReturn(Mockito.mock(ManagedChannel.class)); + + createMtlsPoolAndRotateCert(ChannelPoolSettings.staticallySized(2), channelFactory, executor); + pool.refresh(); + ArgumentCaptor refillTask = ArgumentCaptor.forClass(Runnable.class); + Mockito.verify(executor).execute(refillTask.capture()); + + // Checked and unchecked failures are both handled by expand(); the pool stays usable at its + // reduced size. + FakeLogHandler logHandler = new FakeLogHandler(); + ChannelPool.LOG.addHandler(logHandler); + try { + refillTask.getValue().run(); + assertThat(pool.entries.get()).hasSize(1); + + refillTask.getValue().run(); + assertThat(pool.entries.get()).hasSize(1); + } finally { + ChannelPool.LOG.removeHandler(logHandler); + } + assertThat(logHandler.getAllMessages()) + .containsExactly("Failed to add channel", "Failed to add channel"); + + refillTask.getValue().run(); + assertThat(pool.entries.get()).hasSize(2); + } + + @Test + void refill_failureAfterPartialProgress_keepsCreatedChannels() throws IOException { + ScheduledExecutorService executor = mockExecutor(); + ManagedChannel refilled = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn( + Mockito.mock(ManagedChannel.class), + Mockito.mock(ManagedChannel.class), + Mockito.mock(ManagedChannel.class), + Mockito.mock(ManagedChannel.class)) + .thenThrow(new IOException("Transient failure on second sub-channel")) + .thenThrow(new IOException("Transient failure on third sub-channel")) + .thenReturn(refilled) + .thenThrow(new RuntimeException("Unchecked failure during refill")); + + createMtlsPoolAndRotateCert(ChannelPoolSettings.staticallySized(3), channelFactory, executor); + pool.refresh(); + assertThat(pool.entries.get()).hasSize(1); + ArgumentCaptor refillTask = ArgumentCaptor.forClass(Runnable.class); + Mockito.verify(executor).execute(refillTask.capture()); + + refillTask.getValue().run(); + + // The channel created before the failure is added to the pool rather than orphaned. + assertThat(pool.entries.get()).hasSize(2); + Mockito.verify(refilled, Mockito.never()).shutdown(); + Mockito.verify(channelFactory, Mockito.times(8)).createSingleChannel(); + } + + @Test + void channelReactiveMTlsRefresh_refillRejectedByExecutor_stillCompletesRefresh() + throws IOException { + ScheduledExecutorService executor = mockExecutor(); + Mockito.doThrow(new RejectedExecutionException("Executor shut down")) + .when(executor) + .execute(Mockito.any(Runnable.class)); + ManagedChannel initial1 = Mockito.mock(ManagedChannel.class); + ManagedChannel initial2 = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(initial1, initial2, Mockito.mock(ManagedChannel.class)) + .thenThrow(new IOException("Transient failure on second sub-channel")); + + createMtlsPoolAndRotateCert(ChannelPoolSettings.staticallySized(2), channelFactory, executor); + long genBefore = pool.getGeneration(); + + pool.refresh(); + + Mockito.verify(executor).execute(Mockito.any(Runnable.class)); + assertThat(pool.entries.get()).hasSize(1); + Mockito.verify(initial1).shutdown(); + Mockito.verify(initial2).shutdown(); + assertThat(pool.getGeneration()).isEqualTo(genBefore + 1); + pool.invalidateDiskFingerprintCache(); + assertThat(pool.shouldRefresh()).isFalse(); + } + + @Test + void refreshAll_runtimeExceptionOrError_doesNotLeakCreatedChannels() throws IOException { + ManagedChannel initial1 = Mockito.mock(ManagedChannel.class); + ManagedChannel initial2 = Mockito.mock(ManagedChannel.class); + ManagedChannel createdBeforeRuntimeEx = Mockito.mock(ManagedChannel.class); + ManagedChannel createdBeforeError = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = + Mockito.mock(ChannelFactory.class, Mockito.withSettings().withoutAnnotations()); + + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(initial1, initial2) + .thenReturn(createdBeforeRuntimeEx) + .thenThrow(new RuntimeException("Unchecked runtime exception")) + .thenReturn(createdBeforeError) + .thenThrow(new AssertionError("Simulated Error during refresh")); + + pool = ChannelPool.create(ChannelPoolSettings.staticallySized(2), channelFactory, null, null); + + // Case 1: RuntimeException on channel 1 after creating channel 0 -> caught as Exception, + // partial progress committed + boolean allCreated = pool.refreshAll(); + assertThat(allCreated).isFalse(); + Mockito.verify(initial1).shutdown(); + + // Case 2: Error on channel 1 after creating channel 0 -> aborts, finally block must shut down + // createdBeforeError + assertThrows(AssertionError.class, () -> pool.refreshAll()); + Mockito.verify(createdBeforeError).shutdown(); + } + + @Test + void refresh_onShutdownPool_noOpsAndCreatesNoChannels() throws IOException { + ManagedChannel channel1 = mock(ManagedChannel.class); + ManagedChannel channel2 = mock(ManagedChannel.class); + ChannelFactory channelFactory = + Mockito.mock(ChannelFactory.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(channelFactory.createSingleChannel()).thenReturn(channel1, channel2); + + pool = ChannelPool.create(ChannelPoolSettings.staticallySized(1), channelFactory, null, null); + Mockito.verify(channelFactory, Mockito.times(1)).createSingleChannel(); + + pool.shutdown(); + assertThat(pool.isShutdown()).isTrue(); + + // Invoking refresh or refreshAll on shut down pool must no-op and never create new subchannels + pool.refresh(); + boolean refreshed = pool.refreshAll(); + assertThat(refreshed).isFalse(); + Mockito.verify(channelFactory, Mockito.times(1)).createSingleChannel(); + assertThat(pool.isShutdown()).isTrue(); + } + + @Test + void refreshAll_doesNotIncrementGeneration() throws IOException { + ManagedChannel channel1 = mock(ManagedChannel.class); + ManagedChannel channel2 = mock(ManagedChannel.class); + ChannelFactory channelFactory = + Mockito.mock(ChannelFactory.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(channelFactory.createSingleChannel()).thenReturn(channel1, channel2); + + pool = ChannelPool.create(ChannelPoolSettings.staticallySized(1), channelFactory, null, null); + assertThat(pool.getGeneration()).isEqualTo(0); + + // Only a switch to a new certificate advances the generation. + assertThat(pool.refreshAll()).isTrue(); + Mockito.verify(channel1).shutdown(); + assertThat(pool.getGeneration()).isEqualTo(0); + } + + /** + * Creates an mTLS pool with preemptive refresh enabled and returns the scheduled periodic refresh + * task. + */ + private Runnable createPreemptiveRefreshMtlsPool(int size, ChannelFactory channelFactory) + throws IOException { + return createPreemptiveRefreshMtlsPool(size, channelFactory, mockExecutor()); + } + + private Runnable createPreemptiveRefreshMtlsPool( + int size, ChannelFactory channelFactory, ScheduledExecutorService executor) + throws IOException { + List refreshTasks = new ArrayList<>(); + Mockito.doAnswer( + invocation -> { + refreshTasks.add(invocation.getArgument(0)); + return Mockito.mock( + ScheduledFuture.class, Mockito.withSettings().withoutAnnotations()); + }) + .when(executor) + .scheduleAtFixedRate( + Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + writeCert("client_cert.pem"); + pool = + new ChannelPool( + ChannelPoolSettings.staticallySized(size).toBuilder() + .setPreemptiveRefreshEnabled(true) + .build(), + channelFactory, + FixedExecutorProvider.create(executor), + tempCert.toString()); + assertThat(refreshTasks).hasSize(1); + return refreshTasks.get(0); + } + + private String readCertFingerprint() { + return new CertificateRotationTracker(tempCert.toString()).readDiskFingerprint(); + } + + @Test + void preemptiveRefresh_withoutRotation_doesNotIncrementGeneration() throws IOException { + ManagedChannel initial = Mockito.mock(ManagedChannel.class); + ManagedChannel refreshed = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()).thenReturn(initial, refreshed); + Runnable preemptiveRefresh = createPreemptiveRefreshMtlsPool(1, channelFactory); + + FakeLogHandler logHandler = new FakeLogHandler(); + Level originalLevel = ChannelPool.LOG.getLevel(); + ChannelPool.LOG.setLevel(Level.FINE); + ChannelPool.LOG.addHandler(logHandler); + try { + pool.invalidateDiskFingerprintCache(); + preemptiveRefresh.run(); + } finally { + ChannelPool.LOG.removeHandler(logHandler); + ChannelPool.LOG.setLevel(originalLevel); + } + + // The channels are replaced, but the certificate did not change. + Mockito.verify(initial).shutdown(); + assertThat(pool.getGeneration()).isEqualTo(0); + assertThat(logHandler.getAllMessages()).contains("Refreshing all channels"); + assertThat(String.join("\n", logHandler.getAllMessages())) + .doesNotContain("Channel pool switched to certificate"); + } + + @Test + void preemptiveRefresh_pickingUpRotatedCert_incrementsGeneration() throws IOException { + ManagedChannel initial = Mockito.mock(ManagedChannel.class); + ManagedChannel rotated = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()).thenReturn(initial, rotated); + Runnable preemptiveRefresh = createPreemptiveRefreshMtlsPool(1, channelFactory); + + pool.invalidateDiskFingerprintCache(); + writeCert("root_cert.pem"); + preemptiveRefresh.run(); + + Mockito.verify(initial).shutdown(); + assertThat(pool.getGeneration()).isEqualTo(1); + pool.invalidateDiskFingerprintCache(); + assertThat(pool.shouldRefresh()).isFalse(); + } + + @Test + void preemptiveRefresh_whenCertUnreadable_skipsRefreshAndLogs() throws IOException { + ManagedChannel initial = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()).thenReturn(initial); + Runnable preemptiveRefresh = createPreemptiveRefreshMtlsPool(1, channelFactory); + + // An empty certificate file is what a reader sees while the certificate is being rewritten. + Files.write(tempCert, new byte[0]); + FakeLogHandler logHandler = new FakeLogHandler(); + Level originalLevel = ChannelPool.LOG.getLevel(); + ChannelPool.LOG.setLevel(Level.FINE); + ChannelPool.LOG.addHandler(logHandler); + try { + pool.invalidateDiskFingerprintCache(); + preemptiveRefresh.run(); + } finally { + ChannelPool.LOG.removeHandler(logHandler); + ChannelPool.LOG.setLevel(originalLevel); + } + + Mockito.verify(channelFactory, Mockito.times(1)).createSingleChannel(); + Mockito.verify(initial, Mockito.never()).shutdown(); + assertThat(pool.getGeneration()).isEqualTo(0); + assertThat(String.join("\n", logHandler.getAllMessages())) + .contains("Skipping pre-emptive channel refresh"); + assertThat(logHandler.getAllMessages()).doesNotContain("Refreshing all channels"); + } + + @Test + void preemptiveRefresh_partialFailureOnCertChange_dropsOldChannelsAndCompletesSwitch() + throws IOException { + ScheduledExecutorService executor = mockExecutor(); + ManagedChannel initial1 = Mockito.mock(ManagedChannel.class); + ManagedChannel initial2 = Mockito.mock(ManagedChannel.class); + ManagedChannel rotated1 = Mockito.mock(ManagedChannel.class); + ManagedChannel refilled = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(initial1, initial2) + .thenReturn(rotated1) + .thenThrow(new IOException("Transient failure on second sub-channel")) + .thenReturn(refilled); + Runnable preemptiveRefresh = createPreemptiveRefreshMtlsPool(2, channelFactory, executor); + + pool.invalidateDiskFingerprintCache(); + writeCert("root_cert.pem"); + preemptiveRefresh.run(); + + // The channel that failed to refresh still uses the old certificate, so it is dropped and + // every channel left in the pool uses the new certificate. + Mockito.verify(initial1).shutdown(); + Mockito.verify(initial2).shutdown(); + assertThat(pool.entries.get()).hasSize(1); + assertThat(pool.getGeneration()).isEqualTo(1); + pool.invalidateDiskFingerprintCache(); + assertThat(pool.shouldRefresh()).isFalse(); + + // The pool is refilled to its size before the refresh. + ArgumentCaptor refillTask = ArgumentCaptor.forClass(Runnable.class); + Mockito.verify(executor).execute(refillTask.capture()); + refillTask.getValue().run(); + assertThat(pool.entries.get()).hasSize(2); + Mockito.verify(refilled, Mockito.never()).shutdown(); + } + + @Test + void preemptiveRefresh_allChannelsFailOnCertChange_leavesRotationPending() throws IOException { + ScheduledExecutorService executor = mockExecutor(); + ManagedChannel initial1 = Mockito.mock(ManagedChannel.class); + ManagedChannel initial2 = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(initial1, initial2) + .thenThrow(new IOException("Transient failure")); + Runnable preemptiveRefresh = createPreemptiveRefreshMtlsPool(2, channelFactory, executor); + List entriesBefore = pool.entries.get(); + + pool.invalidateDiskFingerprintCache(); + writeCert("root_cert.pem"); + preemptiveRefresh.run(); + + // No channel could be recreated, so the pool is left unchanged and the rotation stays pending + // for the next periodic or reactive refresh. + assertThat(pool.entries.get()).containsExactlyElementsIn(entriesBefore).inOrder(); + Mockito.verify(initial1, Mockito.never()).shutdown(); + Mockito.verify(initial2, Mockito.never()).shutdown(); + assertThat(pool.getGeneration()).isEqualTo(0); + pool.invalidateDiskFingerprintCache(); + assertThat(pool.shouldRefresh()).isTrue(); + Mockito.verify(executor, Mockito.never()).execute(Mockito.any(Runnable.class)); + } + + @Test + void preemptiveRefresh_partialFailureWithoutCertChange_keepsOldChannel() throws IOException { + ScheduledExecutorService executor = mockExecutor(); + ManagedChannel initial1 = Mockito.mock(ManagedChannel.class); + ManagedChannel initial2 = Mockito.mock(ManagedChannel.class); + ManagedChannel refreshed1 = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(initial1, initial2) + .thenReturn(refreshed1) + .thenThrow(new IOException("Transient failure on second sub-channel")); + Runnable preemptiveRefresh = createPreemptiveRefreshMtlsPool(2, channelFactory, executor); + ChannelPool.Entry initialEntry2 = pool.entries.get().get(1); + + pool.invalidateDiskFingerprintCache(); + preemptiveRefresh.run(); + + // The certificate did not change, so the channel that failed to refresh keeps its slot. + Mockito.verify(initial1).shutdown(); + Mockito.verify(initial2, Mockito.never()).shutdown(); + assertThat(pool.entries.get()).hasSize(2); + assertThat(pool.entries.get().get(1)).isSameInstanceAs(initialEntry2); + assertThat(pool.getGeneration()).isEqualTo(0); + Mockito.verify(executor, Mockito.never()).execute(Mockito.any(Runnable.class)); + } + + /** Waits until {@code thread} is blocked waiting for a monitor lock. */ + private static void awaitBlocked(Thread thread) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (thread.getState() != Thread.State.BLOCKED) { + assertThat(System.nanoTime()).isLessThan(deadline); + Thread.sleep(1); + } + } + + /** + * Starts a reactive refresh that holds the pool lock while it recreates the channel on the + * rotated certificate, runs {@code waiter} on another thread until it blocks on the lock, and + * rotates the certificate again before letting the first refresh complete. A waiter that re-read + * the certificate would see the second rotation and refresh again. + */ + private void runWhileConcurrentSwitchHoldsLock( + CountDownLatch switchStarted, CountDownLatch releaseSwitch, Runnable waiter) + throws Exception { + Thread switchingThread = new Thread(pool::refresh); + Thread waitingThread = new Thread(waiter); + try { + switchingThread.start(); + assertThat(switchStarted.await(5, TimeUnit.SECONDS)).isTrue(); + waitingThread.start(); + awaitBlocked(waitingThread); + writeCert("client_cert.pem"); + } finally { + releaseSwitch.countDown(); + } + switchingThread.join(5000); + waitingThread.join(5000); + assertThat(switchingThread.isAlive()).isFalse(); + assertThat(waitingThread.isAlive()).isFalse(); + } + + @Test + void refresh_concurrentSwitchWhileWaitingForLock_skipsWithoutReadingDisk() throws Exception { + CountDownLatch switchStarted = new CountDownLatch(1); + CountDownLatch releaseSwitch = new CountDownLatch(1); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(Mockito.mock(ManagedChannel.class)) + .thenAnswer( + invocation -> { + switchStarted.countDown(); + releaseSwitch.await(); + return Mockito.mock(ManagedChannel.class); + }) + .thenReturn(Mockito.mock(ManagedChannel.class)); + createMtlsPoolAndRotateCert( + ChannelPoolSettings.staticallySized(1), channelFactory, mockExecutor()); + + runWhileConcurrentSwitchHoldsLock(switchStarted, releaseSwitch, pool::refresh); + + // Only the initial channel and the concurrent switch created channels; the waiter skipped. + Mockito.verify(channelFactory, Mockito.times(2)).createSingleChannel(); + assertThat(pool.getGeneration()).isEqualTo(1); + // The second rotation is still detected for the next UNAUTHENTICATED failure. + pool.invalidateDiskFingerprintCache(); + assertThat(pool.shouldRefresh()).isTrue(); + } + + @Test + void preemptiveRefresh_concurrentSwitchWhileWaitingForLock_skipsRefresh() throws Exception { + CountDownLatch switchStarted = new CountDownLatch(1); + CountDownLatch releaseSwitch = new CountDownLatch(1); + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(Mockito.mock(ManagedChannel.class)) + .thenAnswer( + invocation -> { + switchStarted.countDown(); + releaseSwitch.await(); + return Mockito.mock(ManagedChannel.class); + }) + .thenReturn(Mockito.mock(ManagedChannel.class)); + Runnable preemptiveRefresh = createPreemptiveRefreshMtlsPool(1, channelFactory); + pool.invalidateDiskFingerprintCache(); + writeCert("root_cert.pem"); + + runWhileConcurrentSwitchHoldsLock(switchStarted, releaseSwitch, preemptiveRefresh); + + // The channels were just recreated by the concurrent switch, so the periodic refresh skipped. + Mockito.verify(channelFactory, Mockito.times(2)).createSingleChannel(); + assertThat(pool.getGeneration()).isEqualTo(1); + pool.invalidateDiskFingerprintCache(); + assertThat(pool.shouldRefresh()).isTrue(); + } + + @Test + void refresh_whenCertUnchanged_noOpsAndDoesNotIncrementGeneration() throws IOException { + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(Mockito.mock(ManagedChannel.class), Mockito.mock(ManagedChannel.class)); + writeCert("client_cert.pem"); + pool = + new ChannelPool( + ChannelPoolSettings.staticallySized(1), + channelFactory, + FixedExecutorProvider.create(mockExecutor()), + tempCert.toString()); + + pool.invalidateDiskFingerprintCache(); + pool.refresh(); + + assertThat(pool.getGeneration()).isEqualTo(0); + Mockito.verify(channelFactory, Mockito.times(1)).createSingleChannel(); + } + + @Test + void refresh_onRotation_logsNewCertificateFingerprint() throws IOException { + ChannelFactory channelFactory = mockChannelFactory(); + Mockito.when(channelFactory.createSingleChannel()) + .thenReturn(Mockito.mock(ManagedChannel.class), Mockito.mock(ManagedChannel.class)); + writeCert("client_cert.pem"); + String oldFingerprint = readCertFingerprint(); + createMtlsPoolAndRotateCert( + ChannelPoolSettings.staticallySized(1), channelFactory, mockExecutor()); + String newFingerprint = readCertFingerprint(); + assertThat(newFingerprint).isNotEqualTo(oldFingerprint); + + FakeLogHandler logHandler = new FakeLogHandler(); + Level originalLevel = ChannelPool.LOG.getLevel(); + ChannelPool.LOG.setLevel(Level.FINE); + ChannelPool.LOG.addHandler(logHandler); + try { + pool.refresh(); + } finally { + ChannelPool.LOG.removeHandler(logHandler); + ChannelPool.LOG.setLevel(originalLevel); + } + + assertThat(pool.getGeneration()).isEqualTo(1); + assertThat(logHandler.getAllMessages()) + .contains("Channel pool switched to certificate with fingerprint: " + newFingerprint); + assertThat(String.join("\n", logHandler.getAllMessages())).doesNotContain(oldFingerprint); + } + + @Test + void shutdown_interruptsInProgressRefresh() throws Exception { + CountDownLatch refreshStarted = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicBoolean refreshInterrupted = new AtomicBoolean(); + AtomicInteger createdChannels = new AtomicInteger(); + ChannelFactory channelFactory = + () -> { + if (createdChannels.getAndIncrement() > 0) { + // The preemptive refresh blocks while creating its replacement channel. + refreshStarted.countDown(); + try { + release.await(); + } catch (InterruptedException e) { + refreshInterrupted.set(true); + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while creating channel", e); + } + } + return Mockito.mock(ManagedChannel.class); + }; + ScheduledExecutorService realExecutor = Executors.newSingleThreadScheduledExecutor(); + ScheduledExecutorService executor = mockExecutor(); + Mockito.doAnswer( + invocation -> + realExecutor.schedule( + (Runnable) invocation.getArgument(0), 0, TimeUnit.MILLISECONDS)) + .when(executor) + .scheduleAtFixedRate( + Mockito.any(Runnable.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + try { + pool = + new ChannelPool( + ChannelPoolSettings.staticallySized(1).toBuilder() + .setPreemptiveRefreshEnabled(true) + .build(), + channelFactory, + FixedExecutorProvider.create(executor), + null); + assertThat(refreshStarted.await(5, TimeUnit.SECONDS)).isTrue(); + + // The refresh holds the pool's write lock; shutdown must interrupt it rather than wait. + Assertions.assertTimeoutPreemptively(Duration.ofSeconds(5), () -> pool.shutdown()); + + assertThat(refreshInterrupted.get()).isTrue(); + assertThat(pool.isShutdown()).isTrue(); + } finally { + release.countDown(); + realExecutor.shutdownNow(); + } + } + @Test void channelRefreshShouldSwapChannels() throws IOException { ManagedChannel underlyingChannel1 = mock(ManagedChannel.class); @@ -450,7 +1470,8 @@ void channelRefreshShouldSwapChannels() throws IOException { .setPreemptiveRefreshEnabled(true) .build(), channelFactory, - provider); + provider, + null); Mockito.reset(underlyingChannel1); pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); @@ -459,7 +1480,7 @@ void channelRefreshShouldSwapChannels() throws IOException { .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); // swap channel - pool.refresh(); + pool.refreshAll(); pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); @@ -467,6 +1488,37 @@ void channelRefreshShouldSwapChannels() throws IOException { .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); } + @Test + void testRefreshWithNullWorkloadCertPathSwapsChannel() throws IOException { + ScheduledExecutorService executor = + Mockito.mock(ScheduledExecutorService.class, Mockito.withSettings().withoutAnnotations()); + FixedExecutorProvider provider = FixedExecutorProvider.create(executor); + ManagedChannel underlyingChannel1 = Mockito.mock(ManagedChannel.class); + ManagedChannel underlyingChannel2 = Mockito.mock(ManagedChannel.class); + FakeChannelFactory channelFactory = + new FakeChannelFactory(ImmutableList.of(underlyingChannel1, underlyingChannel2)); + pool = + new ChannelPool( + ChannelPoolSettings.staticallySized(1).toBuilder() + .setPreemptiveRefreshEnabled(true) + .build(), + channelFactory, + provider, + null); + Mockito.reset(underlyingChannel1); + + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + Mockito.verify(underlyingChannel1, Mockito.only()) + .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); + + // Calling refresh() when workloadCertPath is null should fall back to refreshAll() + pool.refresh(); + + pool.newCall(FakeMethodDescriptor.create(), CallOptions.DEFAULT); + Mockito.verify(underlyingChannel2, Mockito.only()) + .newCall(Mockito.>any(), Mockito.any(CallOptions.class)); + } + @Test void channelCountShouldNotChangeWhenOutstandingRpcsAreWithinLimits() throws Exception { ScheduledExecutorService executor = @@ -486,7 +1538,8 @@ void channelCountShouldNotChangeWhenOutstandingRpcsAreWithinLimits() throws Exce .setMaxRpcsPerChannel(2) .build(), channelFactory, - provider); + provider, + null); assertThat(pool.entries.get()).hasSize(2); // Start the minimum number of @@ -553,7 +1606,8 @@ void customResizeDeltaIsRespected() throws Exception { .setMaxResizeDelta(5) .build(), channelFactory, - provider); + provider, + null); assertThat(pool.entries.get()).hasSize(2); // Add 20 RPCs to push expansion @@ -586,7 +1640,8 @@ void removedIdleChannelsAreShutdown() throws Exception { .setMaxRpcsPerChannel(2) .build(), channelFactory, - provider); + provider, + null); assertThat(pool.entries.get()).hasSize(2); // With no outstanding RPCs, the pool should shrink @@ -614,7 +1669,8 @@ void removedActiveChannelsAreShutdown() throws Exception { .setMaxRpcsPerChannel(2) .build(), channelFactory, - provider); + provider, + null); assertThat(pool.entries.get()).hasSize(2); // Start 2 RPCs @@ -652,7 +1708,7 @@ void testReleasingClientCallCancelEarly() throws IOException { Mockito.when(fakeChannel.newCall(Mockito.any(), Mockito.any())).thenReturn(mockClientCall); ChannelPoolSettings channelPoolSettings = ChannelPoolSettings.staticallySized(1); ChannelFactory factory = new FakeChannelFactory(ImmutableList.of(fakeChannel)); - pool = ChannelPool.create(channelPoolSettings, factory, null); + pool = ChannelPool.create(channelPoolSettings, factory, null, null); EndpointContext endpointContext = Mockito.mock(EndpointContext.class, Mockito.withSettings().withoutAnnotations()); @@ -717,7 +1773,8 @@ void repeatedResizingLogsWarningOnExpand() throws Exception { .setMaxChannelCount(10) .build(), channelFactory, - provider); + provider, + null); assertThat(pool.entries.get()).hasSize(1); FakeLogHandler logHandler = new FakeLogHandler(); @@ -769,7 +1826,8 @@ void repeatedResizingLogsWarningOnShrink() throws Exception { .setMaxChannelCount(10) .build(), channelFactory, - provider); + provider, + null); assertThat(pool.entries.get()).hasSize(10); FakeLogHandler logHandler = new FakeLogHandler(); @@ -805,7 +1863,7 @@ void testDoubleRelease() throws Exception { ChannelPoolSettings channelPoolSettings = ChannelPoolSettings.staticallySized(1); ChannelFactory factory = new FakeChannelFactory(ImmutableList.of(fakeChannel)); - pool = ChannelPool.create(channelPoolSettings, factory, null); + pool = ChannelPool.create(channelPoolSettings, factory, null, null); EndpointContext endpointContext = Mockito.mock(EndpointContext.class, Mockito.withSettings().withoutAnnotations()); @@ -843,7 +1901,8 @@ void testDoubleRelease() throws Exception { // Ensure that the channel pool properly logged the double call and kept the refCount correct assertThat(logHandler.getAllMessages()) .contains( - "Call is being closed more than once. Please make sure that onClose() is not being manually called."); + "Call is being closed more than once. Please make sure that onClose() is not being" + + " manually called."); assertThat(pool.entries.get()).hasSize(1); ChannelPool.Entry entry = pool.entries.get().get(0); assertThat(entry.outstandingRpcs.get()).isEqualTo(0); @@ -879,7 +1938,8 @@ void minChannelsClampedToMaxChannelCountUnderHighLoad() throws Exception { .setMaxChannelCount(5) .build(), channelFactory, - provider); + provider, + null); assertThat(pool.entries.get()).hasSize(1); // Add 20 RPCs, which would require 10 channels (20/2) @@ -914,7 +1974,8 @@ void maxChannelsClampedToMinChannelCountUnderLowLoad() throws Exception { .setMaxChannelCount(10) .build(), channelFactory, - provider); + provider, + null); assertThat(pool.entries.get()).hasSize(5); // With no outstanding RPCs, the pool should want to shrink to 0 @@ -924,4 +1985,272 @@ void maxChannelsClampedToMinChannelCountUnderLowLoad() throws Exception { // Should be clamped to minChannelCount = 3 assertThat(pool.entries.get()).hasSize(3); } + + @Test + void shouldRefresh_doesNotCacheNegativeResultAndDetectsSubsequentRotationImmediately() + throws IOException { + ManagedChannel initial = Mockito.mock(ManagedChannel.class); + ManagedChannel rotated = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = + Mockito.mock(ChannelFactory.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(channelFactory.createSingleChannel()).thenReturn(initial, rotated); + + tempCert = Files.createTempFile("cert", ".pem"); + Path clientCert = Paths.get("src", "test", "resources", "client_cert.pem"); + Files.copy(clientCert, tempCert, StandardCopyOption.REPLACE_EXISTING); + + pool = + ChannelPool.create( + ChannelPoolSettings.staticallySized(1), channelFactory, null, tempCert.toString()); + + // First check returns false (unchanged disk cert) + assertThat(pool.shouldRefresh()).isFalse(); + + // Immediately rotate cert on disk WITHOUT invalidating the 1-second cache + Path rootCert = Paths.get("src", "test", "resources", "root_cert.pem"); + Files.copy(rootCert, tempCert, StandardCopyOption.REPLACE_EXISTING); + + // Must immediately detect rotation because negative/unchanged disk checks are not cached for 1s + assertThat(pool.shouldRefresh()).isTrue(); + + // Refresh should update activeCertFingerprint and clear any cached positive check + pool.refresh(); + assertThat(pool.shouldRefresh()).isFalse(); + } + + @Test + void newCall_whenDelegateThrowsError_releasesEntryAndShutsDownRetiredChannel() + throws IOException { + ManagedChannel initial = Mockito.mock(ManagedChannel.class); + ManagedChannel rotated = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = + Mockito.mock(ChannelFactory.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(channelFactory.createSingleChannel()).thenReturn(initial, rotated); + Mockito.when(initial.newCall(Mockito.any(), Mockito.any())) + .thenThrow(new LinkageError("Simulated native/JNI linkage error")); + + pool = ChannelPool.create(ChannelPoolSettings.staticallySized(1), channelFactory, null, null); + + assertThrows(LinkageError.class, () -> pool.newCall(METHOD_RECOGNIZE, CallOptions.DEFAULT)); + + // Rotating the pool should immediately shut down initial channel because its ref count is 0 + pool.refresh(); + Mockito.verify(initial).shutdown(); + } + + @Test + @SuppressWarnings("unchecked") + void start_whenDelegateThrowsError_releasesEntryAndShutsDownRetiredChannel() throws IOException { + ManagedChannel initial = Mockito.mock(ManagedChannel.class); + ManagedChannel rotated = Mockito.mock(ManagedChannel.class); + ClientCall mockCall = Mockito.mock(ClientCall.class); + ChannelFactory channelFactory = + Mockito.mock(ChannelFactory.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(channelFactory.createSingleChannel()).thenReturn(initial, rotated); + Mockito.when(initial.newCall(Mockito.any(), Mockito.any())).thenReturn((ClientCall) mockCall); + Mockito.doThrow(new AssertionError("Simulated Error in start")) + .when(mockCall) + .start(Mockito.any(), Mockito.any()); + + pool = ChannelPool.create(ChannelPoolSettings.staticallySized(1), channelFactory, null, null); + + ClientCall call = pool.newCall(METHOD_RECOGNIZE, CallOptions.DEFAULT); + // Rotate pool while call is retained + pool.refresh(); + Mockito.verify(initial, Mockito.never()).shutdown(); + + // Calling start() throws Error, which must release the retained entry and trigger shutdown + assertThrows( + AssertionError.class, + () -> call.start(new ClientCall.Listener() {}, new Metadata())); + Mockito.verify(initial).shutdown(); + } + + @Test + @SuppressWarnings("unchecked") + void cancel_whenDelegateThrowsException_releasesEntryAndShutsDownRetiredChannel() + throws IOException { + ManagedChannel initial = Mockito.mock(ManagedChannel.class); + ManagedChannel rotated = Mockito.mock(ManagedChannel.class); + ClientCall mockCall = Mockito.mock(ClientCall.class); + ChannelFactory channelFactory = + Mockito.mock(ChannelFactory.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(channelFactory.createSingleChannel()).thenReturn(initial, rotated); + Mockito.when(initial.newCall(Mockito.any(), Mockito.any())).thenReturn((ClientCall) mockCall); + Mockito.doThrow(new RuntimeException("Simulated cancel exception")) + .when(mockCall) + .cancel(Mockito.any(), Mockito.any()); + + pool = ChannelPool.create(ChannelPoolSettings.staticallySized(1), channelFactory, null, null); + + ClientCall call = pool.newCall(METHOD_RECOGNIZE, CallOptions.DEFAULT); + pool.refresh(); + Mockito.verify(initial, Mockito.never()).shutdown(); + + assertThrows(RuntimeException.class, () -> call.cancel("cancelled", null)); + Mockito.verify(initial).shutdown(); + } + + @Test + @SuppressWarnings("unchecked") + void concurrentStartAndCancel_neverLeaksOrDoubleReleasesEntry() throws Exception { + ManagedChannel initial = Mockito.mock(ManagedChannel.class); + ManagedChannel rotated = Mockito.mock(ManagedChannel.class); + ChannelFactory channelFactory = + Mockito.mock(ChannelFactory.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(channelFactory.createSingleChannel()).thenReturn(initial, rotated); + + Mockito.when(initial.newCall(Mockito.any(), Mockito.any())) + .thenAnswer( + invocation -> + new ClientCall() { + private Listener listener; + private boolean cancelled; + + @Override + public synchronized void start( + Listener responseListener, Metadata headers) { + this.listener = responseListener; + if (cancelled) { + responseListener.onClose(Status.CANCELLED, new Metadata()); + } + } + + @Override + public synchronized void cancel(String message, Throwable cause) { + cancelled = true; + if (listener != null) { + listener.onClose(Status.CANCELLED, new Metadata()); + } + } + + @Override + public void request(int numMessages) {} + + @Override + public void halfClose() {} + + @Override + public void sendMessage(Color message) {} + }); + + pool = ChannelPool.create(ChannelPoolSettings.staticallySized(1), channelFactory, null, null); + + int iterations = 100; + ExecutorService executor = Executors.newFixedThreadPool(2); + try { + for (int i = 0; i < iterations; i++) { + ClientCall call = pool.newCall(METHOD_RECOGNIZE, CallOptions.DEFAULT); + CyclicBarrier barrier = new CyclicBarrier(2); + Future f1 = + executor.submit( + () -> { + try { + barrier.await(); + call.start(new ClientCall.Listener() {}, new Metadata()); + } catch (Exception ignored) { + } + }); + Future f2 = + executor.submit( + () -> { + try { + barrier.await(); + call.cancel("cancel", null); + } catch (Exception ignored) { + } + }); + f1.get(5, TimeUnit.SECONDS); + f2.get(5, TimeUnit.SECONDS); + } + } finally { + executor.shutdownNow(); + } + + // Every start/cancel race must leave the entry with no outstanding RPCs: a leak would leave it + // positive and a double release would make it negative. + assertThat(pool.entries.get().get(0).outstandingRpcs.get()).isEqualTo(0); + + // Rotate pool: initial channel must shut down cleanly, proving outstandingRpcs == 0 (no leaks + // or negative counts) + pool.refresh(); + Mockito.verify(initial).shutdown(); + } + + @Test + void cancel_whenStartedAndSuperCancelThrows_doesNotReleasePrematurelyUntilOnClose() + throws Exception { + ManagedChannel initial = Mockito.mock(ManagedChannel.class); + ManagedChannel replacement = Mockito.mock(ManagedChannel.class); + @SuppressWarnings("unchecked") + ClientCall delegateCall = Mockito.mock(ClientCall.class); + @SuppressWarnings("unchecked") + ArgumentCaptor> listenerCaptor = + ArgumentCaptor.forClass(ClientCall.Listener.class); + Mockito.doThrow(new RuntimeException("cancel failure")) + .when(delegateCall) + .cancel(Mockito.any(), Mockito.any()); + Mockito.when(initial.newCall(Mockito.eq(METHOD_RECOGNIZE), Mockito.any())) + .thenReturn(delegateCall); + + AtomicInteger createCount = new AtomicInteger(0); + pool = + new ChannelPool( + ChannelPoolSettings.staticallySized(1), + () -> createCount.getAndIncrement() == 0 ? initial : replacement, + FixedExecutorProvider.create(Mockito.mock(ScheduledExecutorService.class)), + null); + + ClientCall call = pool.newCall(METHOD_RECOGNIZE, CallOptions.DEFAULT); + call.start(new ClientCall.Listener() {}, new Metadata()); + Mockito.verify(delegateCall).start(listenerCaptor.capture(), Mockito.any()); + + assertThrows(RuntimeException.class, () -> call.cancel("abort", null)); + + // Rotate pool while call is still active (onClose hasn't fired yet): + // initial channel must NOT be shut down yet because call is still active + pool.refreshAll(); + Mockito.verify(initial, Mockito.never()).shutdown(); + + // Once onClose fires, entry is released and initial channel shuts down + listenerCaptor.getValue().onClose(Status.CANCELLED, new Metadata()); + Mockito.verify(initial).shutdown(); + } + + @Test + void start_whenCalledTwice_throwsIllegalStateExceptionAndDoesNotReleaseFirstCallEntry() + throws Exception { + ManagedChannel initial = Mockito.mock(ManagedChannel.class); + ManagedChannel replacement = Mockito.mock(ManagedChannel.class); + @SuppressWarnings("unchecked") + ClientCall delegateCall = Mockito.mock(ClientCall.class); + @SuppressWarnings("unchecked") + ArgumentCaptor> listenerCaptor = + ArgumentCaptor.forClass(ClientCall.Listener.class); + Mockito.when(initial.newCall(Mockito.eq(METHOD_RECOGNIZE), Mockito.any())) + .thenReturn(delegateCall); + + AtomicInteger createCount = new AtomicInteger(0); + pool = + new ChannelPool( + ChannelPoolSettings.staticallySized(1), + () -> createCount.getAndIncrement() == 0 ? initial : replacement, + FixedExecutorProvider.create(Mockito.mock(ScheduledExecutorService.class)), + null); + + ClientCall call = pool.newCall(METHOD_RECOGNIZE, CallOptions.DEFAULT); + call.start(new ClientCall.Listener() {}, new Metadata()); + Mockito.verify(delegateCall).start(listenerCaptor.capture(), Mockito.any()); + + // Duplicate start() must throw IllegalStateException without releasing the entry + assertThrows( + IllegalStateException.class, + () -> call.start(new ClientCall.Listener() {}, new Metadata())); + + pool.refreshAll(); + Mockito.verify(initial, Mockito.never()).shutdown(); + + listenerCaptor.getValue().onClose(Status.OK, new Metadata()); + Mockito.verify(initial).shutdown(); + } } diff --git a/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcCallContextTest.java b/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcCallContextTest.java index e20767fdb8ed..0222d9178078 100644 --- a/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcCallContextTest.java +++ b/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcCallContextTest.java @@ -31,6 +31,7 @@ import static com.google.api.gax.util.TimeConversionTestUtils.testDurationMethod; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.Mockito.mock; @@ -52,6 +53,7 @@ import io.grpc.Metadata.Key; import io.grpc.auth.MoreCallCredentials; import java.io.IOException; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -210,6 +212,20 @@ void testWithLongerTimeout() { .isEqualTo(java.time.Duration.ofSeconds(5)); } + @Test + void testWithNullOrZeroTimeoutClearsExistingTimeout() { + GrpcCallContext ctxWithTimeout = + GrpcCallContext.createDefault().withTimeoutDuration(Duration.ofSeconds(5)); + + // Sanity check + Truth.assertThat(ctxWithTimeout.getTimeoutDuration()).isEqualTo(Duration.ofSeconds(5)); + + Duration nullTimeout = null; + Truth.assertThat(ctxWithTimeout.withTimeoutDuration(nullTimeout).getTimeoutDuration()).isNull(); + Truth.assertThat(ctxWithTimeout.withTimeoutDuration(Duration.ZERO).getTimeoutDuration()) + .isNull(); + } + @Test void testMergeWithNullTimeout() { java.time.Duration timeout = java.time.Duration.ofSeconds(10); @@ -494,4 +510,64 @@ private static Map> createTestExtraHeaders(String... keyVal } return extraHeaders; } + + @Test + public void testEqualsAndHashCode() { + ManagedChannel managedChannel1 = Mockito.mock(ManagedChannel.class); + ManagedChannel managedChannel2 = Mockito.mock(ManagedChannel.class); + + GrpcTransportChannel transportChannel1 = GrpcTransportChannel.create(managedChannel1); + GrpcTransportChannel transportChannel2 = GrpcTransportChannel.create(managedChannel2); + + GrpcCallContext context1 = + GrpcCallContext.createDefault().withTransportChannel(transportChannel1); + GrpcCallContext context2 = + GrpcCallContext.createDefault().withTransportChannel(transportChannel1); + GrpcCallContext context3 = + GrpcCallContext.createDefault().withTransportChannel(transportChannel2); + + assertEquals(context1, context2); + assertEquals(context1.hashCode(), context2.hashCode()); + + assertNotEquals(context1, context3); + } + + @Test + public void testMergeWithCustomChannelClearsTransportChannel() { + ManagedChannel defaultChannel = Mockito.mock(ManagedChannel.class); + ManagedChannel customChannel = Mockito.mock(ManagedChannel.class); + GrpcTransportChannel transportChannel = GrpcTransportChannel.create(defaultChannel); + + GrpcCallContext baseContext = + GrpcCallContext.createDefault().withTransportChannel(transportChannel); + GrpcCallContext overrideContext = GrpcCallContext.of(customChannel, CallOptions.DEFAULT); + + GrpcCallContext mergedContext = (GrpcCallContext) baseContext.merge(overrideContext); + assertEquals(customChannel, mergedContext.getChannel()); + assertNull(mergedContext.getTransportChannel()); + } + + @Test + public void testWithChannelWithCustomChannelClearsTransportChannel() { + ManagedChannel defaultChannel = Mockito.mock(ManagedChannel.class); + ManagedChannel customChannel = Mockito.mock(ManagedChannel.class); + GrpcTransportChannel transportChannel = GrpcTransportChannel.create(defaultChannel); + + GrpcCallContext baseContext = + GrpcCallContext.createDefault().withTransportChannel(transportChannel); + GrpcCallContext updatedContext = baseContext.withChannel(customChannel); + + assertEquals(customChannel, updatedContext.getChannel()); + assertNull(updatedContext.getTransportChannel()); + + // Clearing channel via withChannel(null) also clears transportChannel + GrpcCallContext nullChannelContext = baseContext.withChannel(null); + assertNull(nullChannelContext.getChannel()); + assertNull(nullChannelContext.getTransportChannel()); + + // Merging a cleared context into defaultContext falls back to defaultContext's transportChannel + GrpcCallContext mergedWithNullChannel = (GrpcCallContext) baseContext.merge(nullChannelContext); + assertEquals(defaultChannel, mergedWithNullChannel.getChannel()); + assertEquals(transportChannel, mergedWithNullChannel.getTransportChannel()); + } } diff --git a/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcClientCallsTest.java b/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcClientCallsTest.java index 2aa9279e249f..6877eb1dbe1e 100644 --- a/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcClientCallsTest.java +++ b/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcClientCallsTest.java @@ -125,6 +125,7 @@ void testAffinity() throws IOException { ChannelPool.create( ChannelPoolSettings.staticallySized(2), new FakeChannelFactory(Arrays.asList(channel0, channel1)), + null, null); GrpcCallContext context = defaultCallContext.withChannel(pool); diff --git a/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProviderTest.java b/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProviderTest.java index 0b460d6d8e4f..5ffb6943a4fe 100644 --- a/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProviderTest.java +++ b/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProviderTest.java @@ -48,9 +48,11 @@ import com.google.api.gax.rpc.internal.EnvironmentProvider; import com.google.api.gax.rpc.mtls.AbstractMtlsTransportChannelTest; import com.google.api.gax.rpc.mtls.CertificateBasedAccess; +import com.google.api.gax.rpc.testing.FakeMtlsProvider; import com.google.auth.ApiKeyCredentials; import com.google.auth.Credentials; import com.google.auth.http.AuthHttpConstants; +import com.google.auth.mtls.CertificateSourceUnavailableException; import com.google.auth.mtls.MtlsProvider; import com.google.auth.oauth2.CloudShellCredentials; import com.google.auth.oauth2.ComputeEngineCredentials; @@ -663,7 +665,9 @@ private void createAndCloseTransportChannel(InstantiatingGrpcChannelProvider pro createAndCloseTransportChannel(provider); assertThat(logHandler.getAllMessages()) .contains( - "DirectPath is misconfigured. The DirectPath XDS option was set, but the attemptDirectPath option was not. Please set both the attemptDirectPath and attemptDirectPathXds options."); + "DirectPath is misconfigured. The DirectPath XDS option was set, but the" + + " attemptDirectPath option was not. Please set both the attemptDirectPath and" + + " attemptDirectPathXds options."); InstantiatingGrpcChannelProvider.LOG.removeHandler(logHandler); } @@ -681,8 +685,10 @@ void testLogDirectPathMisconfig_AttemptDirectPathNotSetAndAttemptDirectPathXdsSe createAndCloseTransportChannel(provider); assertThat(logHandler.getAllMessages()) .contains( - "Env var GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS was found and set to TRUE, but DirectPath was not enabled for this client. If this is intended for " - + "this client, please note that this is a misconfiguration and set the attemptDirectPath option as well."); + "Env var GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS was found and set to TRUE, but DirectPath" + + " was not enabled for this client. If this is intended for this client, please" + + " note that this is a misconfiguration and set the attemptDirectPath option as" + + " well."); InstantiatingGrpcChannelProvider.LOG.removeHandler(logHandler); } @@ -877,12 +883,14 @@ public void canUseDirectPath_directPathEnvVarDisabled() throws IOException { @Test public void canUseDirectPath_directPathEnvVarNotSet_attemptDirectPathIsTrue() { System.setProperty("os.name", "Linux"); + EnvironmentProvider envProvider = name -> null; InstantiatingGrpcChannelProvider.Builder builder = InstantiatingGrpcChannelProvider.newBuilder() .setCertificateBasedAccess(certificateBasedAccess) .setAttemptDirectPath(true) .setCredentials(computeEngineCredentials) - .setEndpoint(DEFAULT_ENDPOINT); + .setEndpoint(DEFAULT_ENDPOINT) + .setEnvProvider(envProvider); InstantiatingGrpcChannelProvider provider = new InstantiatingGrpcChannelProvider(builder, GCE_PRODUCTION_NAME_AFTER_2016); Truth.assertThat(provider.canUseDirectPath()).isTrue(); @@ -891,12 +899,14 @@ public void canUseDirectPath_directPathEnvVarNotSet_attemptDirectPathIsTrue() { @Test public void canUseDirectPath_directPathEnvVarNotSet_attemptDirectPathIsFalse() { System.setProperty("os.name", "Linux"); + EnvironmentProvider envProvider = name -> null; InstantiatingGrpcChannelProvider.Builder builder = InstantiatingGrpcChannelProvider.newBuilder() .setCertificateBasedAccess(certificateBasedAccess) .setAttemptDirectPath(false) .setCredentials(computeEngineCredentials) - .setEndpoint(DEFAULT_ENDPOINT); + .setEndpoint(DEFAULT_ENDPOINT) + .setEnvProvider(envProvider); InstantiatingGrpcChannelProvider provider = new InstantiatingGrpcChannelProvider(builder, GCE_PRODUCTION_NAME_AFTER_2016); Truth.assertThat(provider.canUseDirectPath()).isFalse(); @@ -1201,7 +1211,8 @@ void createS2ASecuredChannelCredentials_bothS2AAddressesNull_returnsNull() { assertThat(provider.createS2ASecuredChannelCredentials()).isNotNull(); assertThat(logHandler.getAllMessages()) .contains( - "Cannot establish an mTLS connection to S2A because autoconfig endpoint did not return a mtls address to reach S2A."); + "Cannot establish an mTLS connection to S2A because autoconfig endpoint did not return" + + " a mtls address to reach S2A."); InstantiatingGrpcChannelProvider.LOG.removeHandler(logHandler); } @@ -1247,7 +1258,8 @@ void createS2ASecuredChannelCredentials_returnsPlaintextToS2AS2AChannelCredentia assertThat(provider.createS2ASecuredChannelCredentials()).isNotNull(); assertThat(logHandler.getAllMessages()) .contains( - "Cannot establish an mTLS connection to S2A because MTLS to MDS credentials do not exist on filesystem, falling back to plaintext connection to S2A"); + "Cannot establish an mTLS connection to S2A because MTLS to MDS credentials do not" + + " exist on filesystem, falling back to plaintext connection to S2A"); InstantiatingGrpcChannelProvider.LOG.removeHandler(logHandler); } @@ -1342,6 +1354,140 @@ void testSettingBackgroundExecutor() { assertThat(provider.getBackgroundExecutor()).isEqualTo(mockExecutor); } + @Test + void createChannel_whenDirectPathEnabled_ignoresWorkloadCertPath() throws Exception { + System.setProperty("os.name", "Linux"); + EnvironmentProvider envProvider = + mock(EnvironmentProvider.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when( + envProvider.getenv( + InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH)) + .thenReturn("false"); + CertificateBasedAccess mtlsCertificateBasedAccess = + mock(CertificateBasedAccess.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(mtlsCertificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + Mockito.when(mtlsCertificateBasedAccess.getWorkloadCertPath()) + .thenReturn("/path/to/workload/cert.pem"); + MtlsProvider mtlsProvider = + new FakeMtlsProvider(FakeMtlsProvider.createTestMtlsKeyStore(), "", false); + + InstantiatingGrpcChannelProvider.Builder builder = + InstantiatingGrpcChannelProvider.newBuilder() + .setCertificateBasedAccess(mtlsCertificateBasedAccess) + .setMtlsProvider(mtlsProvider) + .setAttemptDirectPath(true) + .setCredentials(computeEngineCredentials) + .setEndpoint(DEFAULT_ENDPOINT) + .setEnvProvider(envProvider) + .setHeaderProvider( + mock(HeaderProvider.class, Mockito.withSettings().withoutAnnotations())); + InstantiatingGrpcChannelProvider provider = + new InstantiatingGrpcChannelProvider(builder, GCE_PRODUCTION_NAME_AFTER_2016); + Truth.assertThat(provider.canUseDirectPath()).isTrue(); + + TransportChannel transportChannel = provider.getTransportChannel(); + try { + ChannelPool pool = (ChannelPool) ((GrpcTransportChannel) transportChannel).getChannel(); + assertThat(pool.getWorkloadCertPath()).isNull(); + } finally { + transportChannel.shutdownNow(); + } + } + + @Test + void createChannel_whenMtlsActive_passesWorkloadCertPathToChannelPool() throws Exception { + CertificateBasedAccess mtlsCertificateBasedAccess = + mock(CertificateBasedAccess.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(mtlsCertificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + Mockito.when(mtlsCertificateBasedAccess.getWorkloadCertPath()) + .thenReturn("/path/to/workload/cert.pem"); + MtlsProvider mtlsProvider = + new FakeMtlsProvider(FakeMtlsProvider.createTestMtlsKeyStore(), "", false); + + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder() + .setCertificateBasedAccess(mtlsCertificateBasedAccess) + .setMtlsProvider(mtlsProvider) + .setAttemptDirectPath(false) + .setEndpoint(DEFAULT_ENDPOINT) + .setHeaderProvider( + mock(HeaderProvider.class, Mockito.withSettings().withoutAnnotations())) + .build(); + + TransportChannel transportChannel = provider.getTransportChannel(); + try { + ChannelPool pool = (ChannelPool) ((GrpcTransportChannel) transportChannel).getChannel(); + assertThat(pool.getWorkloadCertPath()).isEqualTo("/path/to/workload/cert.pem"); + } finally { + transportChannel.shutdownNow(); + } + } + + @Test + void createChannelBuilder_whenMtlsActiveAndKeyStoreNull_throwsIOException() { + CertificateBasedAccess mtlsCertificateBasedAccess = + mock(CertificateBasedAccess.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(mtlsCertificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + MtlsProvider mtlsProviderWithNullKeyStore = new FakeMtlsProvider(null, "", false); + + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder() + .setCertificateBasedAccess(mtlsCertificateBasedAccess) + .setMtlsProvider(mtlsProviderWithNullKeyStore) + .setAttemptDirectPath(false) + .setEndpoint(DEFAULT_ENDPOINT) + .setHeaderProvider( + mock(HeaderProvider.class, Mockito.withSettings().withoutAnnotations())) + .build(); + + IOException thrown = assertThrows(IOException.class, provider::createChannelBuilder); + assertThat(thrown).hasMessageThat().contains("Failed to initialize mTLS channel credentials"); + } + + @Test + void createChannelBuilder_whenDefaultMtlsProviderThrowsIOException_propagatesIOException() + throws Exception { + CertificateBasedAccess mtlsCertificateBasedAccess = + mock(CertificateBasedAccess.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(mtlsCertificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + IOException expected = new IOException("Corrupt PEM file"); + Mockito.when(mtlsCertificateBasedAccess.createDefaultMtlsProvider()).thenThrow(expected); + + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder() + .setCertificateBasedAccess(mtlsCertificateBasedAccess) + .setAttemptDirectPath(false) + .setEndpoint(DEFAULT_ENDPOINT) + .setHeaderProvider( + mock(HeaderProvider.class, Mockito.withSettings().withoutAnnotations())) + .build(); + + IOException thrown = assertThrows(IOException.class, provider::createChannelBuilder); + assertThat(thrown).isSameInstanceAs(expected); + } + + @Test + void + createChannelBuilder_whenDefaultMtlsProviderThrowsCertificateSourceUnavailable_fallsBackWithoutMtls() + throws Exception { + CertificateBasedAccess mtlsCertificateBasedAccess = + mock(CertificateBasedAccess.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(mtlsCertificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + Mockito.when(mtlsCertificateBasedAccess.createDefaultMtlsProvider()) + .thenThrow(new CertificateSourceUnavailableException("No certificates on disk")); + + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder() + .setCertificateBasedAccess(mtlsCertificateBasedAccess) + .setAttemptDirectPath(false) + .setEndpoint(DEFAULT_ENDPOINT) + .setHeaderProvider( + mock(HeaderProvider.class, Mockito.withSettings().withoutAnnotations())) + .build(); + + assertThat(provider.createChannelBuilder()).isNotNull(); + } + private static class FakeLogHandler extends Handler { List records = new ArrayList<>(); diff --git a/sdk-platform-java/gax-java/gax-httpjson/BUILD.bazel b/sdk-platform-java/gax-java/gax-httpjson/BUILD.bazel index c4f78edb79a2..e97aedebabfd 100644 --- a/sdk-platform-java/gax-java/gax-httpjson/BUILD.bazel +++ b/sdk-platform-java/gax-java/gax-httpjson/BUILD.bazel @@ -53,6 +53,10 @@ java_library( srcs = glob(["src/test/java/**/*.java"]), javacopts = _JAVA_COPTS, plugins = ["//:auto_value_plugin"], + resources = glob([ + "src/test/resources/com/google/api/gax/httpjson/tls13TestCa.pem", + "src/test/resources/com/google/api/gax/httpjson/tls13TestServerCertAndKey.pem", + ]), visibility = ["//visibility:public"], deps = [":gax_httpjson"] + _COMPILE_DEPS + _TEST_COMPILE_DEPS, ) diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonCallContext.java b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonCallContext.java index daefe8736eda..c5cb09338872 100644 --- a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonCallContext.java +++ b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonCallContext.java @@ -81,6 +81,7 @@ public final class HttpJsonCallContext implements ApiCallContext { private final @Nullable RetrySettings retrySettings; private final @Nullable ImmutableSet retryableCodes; private final EndpointContext endpointContext; + @Nullable private final TransportChannel transportChannel; /** Returns an empty instance. */ public static HttpJsonCallContext createDefault() { @@ -95,6 +96,7 @@ public static HttpJsonCallContext createDefault() { null, null, null, + null, null); } @@ -110,6 +112,7 @@ public static HttpJsonCallContext of(HttpJsonChannel channel, HttpJsonCallOption null, null, null, + null, null); } @@ -124,7 +127,8 @@ private HttpJsonCallContext( @Nullable ApiTracer tracer, @Nullable RetrySettings defaultRetrySettings, @Nullable Set defaultRetryableCodes, - @Nullable EndpointContext endpointContext) { + @Nullable EndpointContext endpointContext, + @Nullable TransportChannel transportChannel) { this.channel = channel; this.callOptions = callOptions; this.timeout = timeout; @@ -140,6 +144,7 @@ private HttpJsonCallContext( // a valid EndpointContext with user configurations after the client has been initialized. this.endpointContext = endpointContext == null ? EndpointContext.getDefaultInstance() : endpointContext; + this.transportChannel = transportChannel; } /** @@ -219,6 +224,12 @@ public HttpJsonCallContext merge(ApiCallContext inputCallContext) { newRetryableCodes = this.retryableCodes; } + TransportChannel newTransportChannel = httpJsonCallContext.transportChannel; + if (newTransportChannel == null + && (httpJsonCallContext.channel == null || httpJsonCallContext.channel.equals(channel))) { + newTransportChannel = this.transportChannel; + } + // The EndpointContext is not updated as there should be no reason for a user // to update this. return new HttpJsonCallContext( @@ -232,7 +243,8 @@ public HttpJsonCallContext merge(ApiCallContext inputCallContext) { newTracer, newRetrySettings, newRetryableCodes, - endpointContext); + endpointContext, + newTransportChannel); } @Override @@ -250,7 +262,24 @@ public HttpJsonCallContext withTransportChannel(TransportChannel inputChannel) { "Expected HttpJsonTransportChannel, got " + inputChannel.getClass().getName()); } HttpJsonTransportChannel transportChannel = (HttpJsonTransportChannel) inputChannel; - return withChannel(transportChannel.getChannel()); + return new HttpJsonCallContext( + transportChannel.getChannel(), + this.callOptions, + this.timeout, + this.streamWaitTimeout, + this.streamIdleTimeout, + this.extraHeaders, + this.options, + this.tracer, + this.retrySettings, + this.retryableCodes, + this.endpointContext, + transportChannel); + } + + @Override + public TransportChannel getTransportChannel() { + return transportChannel; } /** This method is obsolete. Use {@link #withTimeoutDuration(java.time.Duration)} instead. */ @@ -274,7 +303,8 @@ public HttpJsonCallContext withEndpointContext(EndpointContext endpointContext) this.tracer, this.retrySettings, this.retryableCodes, - endpointContext); + endpointContext, + this.transportChannel); } @Override @@ -300,7 +330,8 @@ public HttpJsonCallContext withTimeoutDuration(java.time.Duration timeout) { this.tracer, this.retrySettings, this.retryableCodes, - this.endpointContext); + this.endpointContext, + this.transportChannel); } /** This method is obsolete. Use {@link #getTimeoutDuration()} instead. */ @@ -345,7 +376,8 @@ public HttpJsonCallContext withStreamWaitTimeoutDuration( this.tracer, this.retrySettings, this.retryableCodes, - this.endpointContext); + this.endpointContext, + this.transportChannel); } /** This method is obsolete. Use {@link #getStreamWaitTimeoutDuration()} instead. */ @@ -395,7 +427,8 @@ public HttpJsonCallContext withStreamIdleTimeoutDuration( this.tracer, this.retrySettings, this.retryableCodes, - this.endpointContext); + this.endpointContext, + this.transportChannel); } /** This method is obsolete. Use {@link #getStreamIdleTimeoutDuration()} instead. */ @@ -432,7 +465,8 @@ public ApiCallContext withExtraHeaders(Map> extraHeaders) { this.tracer, this.retrySettings, this.retryableCodes, - this.endpointContext); + this.endpointContext, + this.transportChannel); } @BetaApi("The surface for extra headers is not stable yet and may change in the future.") @@ -456,7 +490,8 @@ public ApiCallContext withOption(Key key, T value) { this.tracer, this.retrySettings, this.retryableCodes, - this.endpointContext); + this.endpointContext, + this.transportChannel); } /** {@inheritDoc} */ @@ -526,7 +561,8 @@ public HttpJsonCallContext withRetrySettings(RetrySettings retrySettings) { this.tracer, retrySettings, this.retryableCodes, - this.endpointContext); + this.endpointContext, + this.transportChannel); } @Override @@ -547,7 +583,8 @@ public HttpJsonCallContext withRetryableCodes(Set retryableCode this.tracer, this.retrySettings, retryableCodes, - this.endpointContext); + this.endpointContext, + this.transportChannel); } public HttpJsonCallContext withChannel(@Nullable HttpJsonChannel newChannel) { @@ -562,7 +599,8 @@ public HttpJsonCallContext withChannel(@Nullable HttpJsonChannel newChannel) { this.tracer, this.retrySettings, this.retryableCodes, - this.endpointContext); + this.endpointContext, + (newChannel != null && newChannel.equals(this.channel)) ? this.transportChannel : null); } public HttpJsonCallContext withCallOptions(HttpJsonCallOptions newCallOptions) { @@ -577,7 +615,8 @@ public HttpJsonCallContext withCallOptions(HttpJsonCallOptions newCallOptions) { this.tracer, this.retrySettings, this.retryableCodes, - this.endpointContext); + this.endpointContext, + this.transportChannel); } @Deprecated @@ -612,7 +651,8 @@ public HttpJsonCallContext withTracer(ApiTracer newTracer) { newTracer, this.retrySettings, this.retryableCodes, - this.endpointContext); + this.endpointContext, + this.transportChannel); } @Override @@ -632,7 +672,8 @@ public boolean equals(@Nullable Object o) { && Objects.equals(this.tracer, that.tracer) && Objects.equals(this.retrySettings, that.retrySettings) && Objects.equals(this.retryableCodes, that.retryableCodes) - && Objects.equals(this.endpointContext, that.endpointContext); + && Objects.equals(this.endpointContext, that.endpointContext) + && Objects.equals(this.transportChannel, that.transportChannel); } @Override @@ -646,6 +687,7 @@ public int hashCode() { tracer, retrySettings, retryableCodes, - endpointContext); + endpointContext, + transportChannel); } } diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonTransportChannel.java b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonTransportChannel.java index 813622b6a97e..fde0673f650d 100644 --- a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonTransportChannel.java +++ b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonTransportChannel.java @@ -64,6 +64,21 @@ public HttpJsonChannel getChannel() { return getManagedChannel(); } + @Override + public void refresh() { + getManagedChannel().refresh(); + } + + @Override + public boolean shouldRefresh() { + return getManagedChannel().shouldRefresh(); + } + + @Override + public long getGeneration() { + return getManagedChannel().getGeneration(); + } + @Override public void shutdown() { getManagedChannel().shutdown(); diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProvider.java b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProvider.java index f0a6aa380d37..0766a8032df1 100644 --- a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProvider.java +++ b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProvider.java @@ -40,7 +40,6 @@ import com.google.api.gax.rpc.mtls.CertificateBasedAccess; import com.google.auth.Credentials; import com.google.auth.mtls.CertificateSourceUnavailableException; -import com.google.auth.mtls.DefaultMtlsProviderFactory; import com.google.auth.mtls.MtlsProvider; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; @@ -50,9 +49,11 @@ import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.ScheduledExecutorService; -import java.util.logging.Level; +import java.util.function.Supplier; import java.util.logging.Logger; +import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; @@ -216,49 +217,116 @@ private NetHttpTransport.Builder configureMtls(NetHttpTransport.Builder builder) // Fall back to standard JDK JSSE if Conscrypt provider is unavailable return builder; } - // Explicitly initialize SSLContext with the Conscrypt provider so that the client certificate - // key managers - // and trust manager factory (TMF) are bound to Conscrypt's TLS implementation (supporting PQC - // key exchange). + // NetHttpTransport.Builder.trustCertificates() in google-http-client initializes with the + // default SunX509 KeyManagerFactory (via SslUtils.getDefaultKeyManagerFactory()), which is not + // supported by Conscrypt. Explicitly initialize SSLContext with the Conscrypt provider so that + // the client certificate key managers and trust manager factory (TMF) are bound to Conscrypt's + // TLS implementation (supporting PQC key exchange). SSLContext sslContext = SSLContext.getInstance("TLS", conscryptProvider); + // The TrustManagerFactory must come from the same provider as the SSLContext. On TLS 1.3, + // Conscrypt passes authType "GENERIC" to the trust manager, which the JDK (SunJSSE) PKIX + // trust manager rejects for CA-issued server certificates that carry a KeyUsage extension + // (e.g. Google front ends), failing the handshake with "Unknown authType: GENERIC". + // Conscrypt's trust manager loads the same default trust store as the JDK. + // The KeyManagerFactory must also come from Conscrypt: since JDK 26 (JDK-8359956) the JDK's + // SunX509 key manager applies algorithm constraints to the client certificate, and when called + // from a Conscrypt handshake it rejects valid certificates (e.g. SHA256withRSA), so no client + // certificate is sent. SslUtils.initSslContext( sslContext, null, - SslUtils.getPkixTrustManagerFactory(), + TrustManagerFactory.getInstance("PKIX", conscryptProvider), mtlsKeyStore, "", - SslUtils.getDefaultKeyManagerFactory()); + KeyManagerFactory.getInstance("PKIX", conscryptProvider)); builder.setSslSocketFactory(sslContext.getSocketFactory()); return builder; } - private HttpJsonTransportChannel createChannel() throws IOException, GeneralSecurityException { - HttpTransport httpTransportToUse = httpTransport; - if (httpTransportToUse == null) { - httpTransportToUse = createHttpTransport(); + private HttpTransport createChannelHttpTransport() throws IOException, GeneralSecurityException { + HttpTransport transport = createHttpTransport(); + if (mtlsProvider != null + && certificateBasedAccess.useMtlsClientCertificate() + && !((NetHttpTransport) transport).isMtls()) { + // mTLS is enabled but the provider returned no client certificate. Fail instead of silently + // using a transport without the certificate, matching InstantiatingGrpcChannelProvider. + // During certificate rotation, this makes RefreshingHttpJsonChannel keep the current + // authenticated transport instead of swapping in one without a client certificate. + throw new IOException("Failed to initialize mTLS HttpTransport"); } + return transport; + } + + private ManagedHttpJsonChannel createSingleManagedChannel() + throws IOException, GeneralSecurityException { + return buildManagedChannel( + httpTransport != null ? httpTransport : createChannelHttpTransport()); + } - // Pass the executor to the ManagedChannel. If no executor was provided (or null), - // the channel will use a default executor for the calls. - ManagedHttpJsonChannel channel = - ManagedHttpJsonChannel.newBuilder() - .setEndpoint(endpoint) - .setExecutor(executor) - .setHttpTransport(httpTransportToUse) - .build(); - - HttpJsonClientInterceptor headerInterceptor = - new HttpJsonHeaderInterceptor(headerProvider.getHeaders()); - - channel = new ManagedHttpJsonInterceptorChannel(channel, new HttpJsonLoggingInterceptor()); - channel = new ManagedHttpJsonInterceptorChannel(channel, headerInterceptor); - if (interceptorProvider != null && interceptorProvider.getInterceptors() != null) { - for (HttpJsonClientInterceptor interceptor : interceptorProvider.getInterceptors()) { - channel = new ManagedHttpJsonInterceptorChannel(channel, interceptor); + private ManagedHttpJsonChannel buildManagedChannel(HttpTransport httpTransportToUse) { + return ManagedHttpJsonChannel.newBuilder() + .setEndpoint(endpoint) + .setExecutor(executor) + .setHttpTransport(httpTransportToUse) + .setManageHttpTransport(httpTransport == null) + .build(); + } + + private HttpJsonTransportChannel createChannel() throws IOException, GeneralSecurityException { + boolean isMtlsActive = + httpTransport == null + && mtlsProvider != null + && certificateBasedAccess.useMtlsClientCertificate(); + String workloadCertPath = isMtlsActive ? certificateBasedAccess.getWorkloadCertPath() : null; + + ManagedHttpJsonChannel baseChannel; + if (workloadCertPath != null) { + Supplier transportFactory = + () -> { + try { + return createChannelHttpTransport(); + } catch (IOException | GeneralSecurityException e) { + throw new RuntimeException("Failed to create mTLS HttpTransport", e); + } + }; + // RefreshingHttpJsonChannel records the baseline certificate fingerprint before creating the + // initial transport, so a rotation during startup is detected on the next auth failure. + try { + baseChannel = + new RefreshingHttpJsonChannel( + transportFactory, this::buildManagedChannel, workloadCertPath); + } catch (RuntimeException e) { + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); + } + if (e.getCause() instanceof GeneralSecurityException) { + throw (GeneralSecurityException) e.getCause(); + } + throw e; } + } else { + baseChannel = createSingleManagedChannel(); } - return HttpJsonTransportChannel.newBuilder().setManagedChannel(channel).build(); + try { + ManagedHttpJsonChannel channel = baseChannel; + + HttpJsonClientInterceptor headerInterceptor = + new HttpJsonHeaderInterceptor(headerProvider.getHeaders()); + + channel = new ManagedHttpJsonInterceptorChannel(channel, new HttpJsonLoggingInterceptor()); + channel = new ManagedHttpJsonInterceptorChannel(channel, headerInterceptor); + if (interceptorProvider != null && interceptorProvider.getInterceptors() != null) { + for (HttpJsonClientInterceptor interceptor : interceptorProvider.getInterceptors()) { + channel = new ManagedHttpJsonInterceptorChannel(channel, interceptor); + } + } + + return HttpJsonTransportChannel.newBuilder().setManagedChannel(channel).build(); + } catch (Throwable t) { + baseChannel.shutdownNow(); + throw t; + } } /** The endpoint to be used for the channel. */ @@ -383,18 +451,23 @@ public InstantiatingHttpJsonChannelProvider build() { if (mtlsProvider == null) { // Attempt to create default MtlsProvider from environment. try { - mtlsProvider = DefaultMtlsProviderFactory.create(); + mtlsProvider = certificateBasedAccess.createDefaultMtlsProvider(); } catch (CertificateSourceUnavailableException e) { // This is okay. Leave mtlsProvider as null so that we will not auto-upgrade // to mTLS endpoints. See https://google.aip.dev/auth/4114. } catch (IOException e) { - LOG.log( - Level.WARNING, - "DefaultMtlsProviderFactory encountered unexpected IOException: " + e.getMessage()); - LOG.log( - Level.WARNING, - "mTLS configuration was detected on the device, but mTLS failed to initialize." - + " Falling back to non-mTLS channel."); + mtlsProvider = + new MtlsProvider() { + @Override + public boolean isAvailable() throws IOException { + throw e; + } + + @Override + public KeyStore getKeyStore() throws IOException { + throw e; + } + }; } } } diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/ManagedHttpJsonChannel.java b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/ManagedHttpJsonChannel.java index 87767bee5c7f..bf79e17d7d89 100644 --- a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/ManagedHttpJsonChannel.java +++ b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/ManagedHttpJsonChannel.java @@ -51,12 +51,35 @@ public class ManagedHttpJsonChannel implements HttpJsonChannel, BackgroundResour private final Executor executor; private final boolean usingDefaultExecutor; private final String endpoint; - private final HttpTransport httpTransport; + private volatile HttpTransport httpTransport; + private final boolean usingDefaultTransport; private final ScheduledExecutorService deadlineScheduledExecutorService; private boolean isTransportShutdown; protected ManagedHttpJsonChannel() { - this(null, true, null, null); + this(null, true, null, null, true); + } + + /** + * Constructor for subclasses that delegate all calls to a wrapped channel. The argument is + * unused; it only distinguishes this overload from {@link #ManagedHttpJsonChannel()}, which would + * otherwise allocate a transport and executor that the wrapper never uses or shuts down. + */ + ManagedHttpJsonChannel(boolean isDelegatingWrapper) { + this.executor = null; + this.usingDefaultExecutor = false; + this.endpoint = null; + this.httpTransport = null; + this.usingDefaultTransport = false; + this.deadlineScheduledExecutorService = null; + } + + /** + * Returns a monotonic generation counter tracking the number of successful refreshes or channel + * rotations performed by this channel. Always {@code 0} for channels that do not refresh. + */ + public long getGeneration() { + return 0; } String getEndpoint() { @@ -68,11 +91,20 @@ HttpTransport getHttpTransport() { return httpTransport; } + /** + * Replaces the transport used by calls created after this method returns. Calls that were already + * created keep using the transport they were created with. + */ + void setHttpTransport(HttpTransport httpTransport) { + this.httpTransport = httpTransport; + } + private ManagedHttpJsonChannel( @Nullable Executor executor, boolean usingDefaultExecutor, @Nullable String endpoint, - @Nullable HttpTransport httpTransport) { + @Nullable HttpTransport httpTransport, + boolean usingDefaultTransport) { this.executor = executor; this.usingDefaultExecutor = usingDefaultExecutor; this.endpoint = endpoint; @@ -82,6 +114,7 @@ private ManagedHttpJsonChannel( new NetHttpTransport.Builder()) .build() : httpTransport; + this.usingDefaultTransport = usingDefaultTransport || httpTransport == null; this.deadlineScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); } @@ -98,6 +131,22 @@ public HttpJsonClientCall newCall( deadlineScheduledExecutorService); } + /** + * Refreshes or recreates the underlying transport of this channel if a certificate rotation has + * been detected. By default, this is a no-op. + */ + public void refresh() { + // No-op: this channel has no certificate to rotate. Overridden by RefreshingHttpJsonChannel. + } + + /** + * Returns true if a certificate rotation has been detected on disk and this channel should be + * refreshed, or false otherwise. Always {@code false} for channels that do not refresh. + */ + public boolean shouldRefresh() { + return false; + } + @VisibleForTesting Executor getExecutor() { return executor; @@ -116,7 +165,9 @@ public synchronized void shutdown() { ((ExecutorService) executor).shutdown(); } deadlineScheduledExecutorService.shutdown(); - httpTransport.shutdown(); + if (usingDefaultTransport) { + httpTransport.shutdown(); + } isTransportShutdown = true; } catch (IOException e) { // TODO: Log this scenario once we implemented the Cloud SDK logging. @@ -158,7 +209,9 @@ public void shutdownNow() { ((ExecutorService) executor).shutdownNow(); } deadlineScheduledExecutorService.shutdownNow(); - httpTransport.shutdown(); + if (usingDefaultTransport) { + httpTransport.shutdown(); + } isTransportShutdown = true; } catch (IOException e) { // TODO: Log this scenario once we implemented the Cloud SDK logging. @@ -205,9 +258,11 @@ public static class Builder { private String endpoint; private HttpTransport httpTransport; private boolean usingDefaultExecutor; + private boolean usingDefaultTransport; private Builder() { this.usingDefaultExecutor = false; + this.usingDefaultTransport = false; } public Builder setExecutor(Executor executor) { @@ -225,6 +280,11 @@ public Builder setHttpTransport(HttpTransport httpTransport) { return this; } + Builder setManageHttpTransport(boolean manageHttpTransport) { + this.usingDefaultTransport = manageHttpTransport; + return this; + } + public ManagedHttpJsonChannel build() { Preconditions.checkNotNull(endpoint); @@ -237,14 +297,8 @@ public ManagedHttpJsonChannel build() { usingDefaultExecutor = true; } - if (httpTransport == null) { - httpTransport = - HttpJsonConscryptUtils.configureConscryptSecurityProvider( - new NetHttpTransport.Builder()) - .build(); - } - - return new ManagedHttpJsonChannel(executor, usingDefaultExecutor, endpoint, httpTransport); + return new ManagedHttpJsonChannel( + executor, usingDefaultExecutor, endpoint, httpTransport, usingDefaultTransport); } } } diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/ManagedHttpJsonInterceptorChannel.java b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/ManagedHttpJsonInterceptorChannel.java index eaaa8c3a7c56..9adbfdd5c329 100644 --- a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/ManagedHttpJsonInterceptorChannel.java +++ b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/ManagedHttpJsonInterceptorChannel.java @@ -29,7 +29,9 @@ */ package com.google.api.gax.httpjson; +import com.google.api.client.http.HttpTransport; import com.google.common.annotations.VisibleForTesting; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import org.jspecify.annotations.NullMarked; @@ -41,22 +43,57 @@ class ManagedHttpJsonInterceptorChannel extends ManagedHttpJsonChannel { ManagedHttpJsonInterceptorChannel( ManagedHttpJsonChannel channel, HttpJsonClientInterceptor interceptor) { - super(); + super(true); this.channel = channel; this.interceptor = interceptor; } + /** {@inheritDoc} */ + @Override + public long getGeneration() { + return channel.getGeneration(); + } + @VisibleForTesting ManagedHttpJsonChannel getChannel() { return channel; } + @Override + String getEndpoint() { + return channel.getEndpoint(); + } + + @Override + @VisibleForTesting + HttpTransport getHttpTransport() { + return channel.getHttpTransport(); + } + + @Override + @VisibleForTesting + Executor getExecutor() { + return channel.getExecutor(); + } + @Override public HttpJsonClientCall newCall( ApiMethodDescriptor methodDescriptor, HttpJsonCallOptions callOptions) { return interceptor.interceptCall(methodDescriptor, callOptions, channel); } + /** {@inheritDoc} */ + @Override + public void refresh() { + channel.refresh(); + } + + /** {@inheritDoc} */ + @Override + public boolean shouldRefresh() { + return channel.shouldRefresh(); + } + @Override public synchronized void shutdown() { channel.shutdown(); diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/RefreshingHttpJsonChannel.java b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/RefreshingHttpJsonChannel.java new file mode 100644 index 000000000000..6d20a69139f8 --- /dev/null +++ b/sdk-platform-java/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/RefreshingHttpJsonChannel.java @@ -0,0 +1,213 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.httpjson; + +import com.google.api.client.http.HttpTransport; +import com.google.api.core.InternalApi; +import com.google.api.gax.rpc.mtls.CertificateRotationTracker; +import com.google.api.gax.rpc.mtls.WorkloadCertificateUtils; +import com.google.common.annotations.VisibleForTesting; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * An implementation of {@link ManagedHttpJsonChannel} that supports dynamic mTLS certificate + * rotation. When the workload certificate on disk changes, {@link #refresh()} replaces the {@link + * HttpTransport} of the underlying channel. Calls that were already created keep using the + * transport they were created with, so in-flight requests are not interrupted. + */ +@InternalApi +public class RefreshingHttpJsonChannel extends ManagedHttpJsonChannel { + + private static final Logger LOG = Logger.getLogger(RefreshingHttpJsonChannel.class.getName()); + + private final CertificateRotationTracker rotationTracker; + private final Supplier transportFactory; + private final String workloadCertPath; + private final ManagedHttpJsonChannel delegate; + private final Object refreshLock = new Object(); + private final AtomicLong generation = new AtomicLong(0); + + /** + * @param transportFactory creates a transport configured with the certificate currently on disk + * @param channelFactory creates the underlying channel from the initial transport + * @param workloadCertPath path of the workload certificate to monitor for rotation + */ + public RefreshingHttpJsonChannel( + Supplier transportFactory, + Function channelFactory, + String workloadCertPath) { + super(true); + this.transportFactory = transportFactory; + this.workloadCertPath = workloadCertPath; + // Record the baseline fingerprint before the initial transport loads the certificate from disk, + // so a rotation between the two steps is detected as a fingerprint change. + this.rotationTracker = + new CertificateRotationTracker(this::getWorkloadCertPath, this::getCertificateFingerprint); + this.delegate = channelFactory.apply(transportFactory.get()); + } + + @VisibleForTesting + String getWorkloadCertPath() { + return workloadCertPath; + } + + @VisibleForTesting + String getCertificateFingerprint(String certPath) { + return WorkloadCertificateUtils.getCertificateFingerprint(certPath); + } + + /** {@inheritDoc} */ + @Override + public boolean shouldRefresh() { + return rotationTracker.shouldRefresh(); + } + + /** {@inheritDoc} */ + @Override + public void refresh() { + // A generation change while waiting for the lock means a concurrent refresh already swapped in + // a transport with a new certificate, so there is no need to read the certificate again. + long generationBeforeLock = generation.get(); + synchronized (refreshLock) { + if (isShutdown()) { + return; + } + if (generation.get() != generationBeforeLock) { + LOG.fine( + "HTTP/JSON channel was already refreshed by a concurrent thread, skipping duplicate" + + " refresh"); + return; + } + String currentDiskFingerprint = rotationTracker.readDiskFingerprint(); + if (currentDiskFingerprint.isEmpty()) { + return; + } + + // Double-check inside refreshLock + if (rotationTracker.isAlreadyActive(currentDiskFingerprint)) { + LOG.fine( + "HTTP/JSON channel was already refreshed by a concurrent thread, skipping duplicate" + + " refresh"); + return; + } + + LOG.info("mTLS certificate rotation detected. Refreshing HTTP/JSON transport."); + + HttpTransport newTransport; + try { + newTransport = transportFactory.get(); + } catch (Exception e) { + LOG.log(Level.WARNING, "Failed to refresh HTTP/JSON transport, keeping old transport", e); + return; + } + // The previous transport is not shut down: calls created before the swap may still be using + // it, and it needs no explicit shutdown because its idle keep-alive connections expire. + delegate.setHttpTransport(newTransport); + // Order matters: swap the transport, then bump generation, then mark the tracker refreshed, + // so any failing RPC that observes the new fingerprint also observes the new generation. + generation.incrementAndGet(); + rotationTracker.markRefreshed(currentDiskFingerprint); + } + } + + /** {@inheritDoc} */ + @Override + public long getGeneration() { + return generation.get(); + } + + @Override + public HttpJsonClientCall newCall( + ApiMethodDescriptor methodDescriptor, HttpJsonCallOptions callOptions) { + return delegate.newCall(methodDescriptor, callOptions); + } + + @Override + Executor getExecutor() { + return delegate.getExecutor(); + } + + @Override + String getEndpoint() { + return delegate.getEndpoint(); + } + + @Override + @VisibleForTesting + HttpTransport getHttpTransport() { + return delegate.getHttpTransport(); + } + + @VisibleForTesting + void invalidateDiskFingerprintCache() { + rotationTracker.invalidateCache(); + } + + @Override + public void shutdown() { + // Serialized with refresh() so that a transport swap cannot race with shutdown. + synchronized (refreshLock) { + delegate.shutdown(); + } + } + + @Override + public boolean isShutdown() { + return delegate.isShutdown(); + } + + @Override + public boolean isTerminated() { + return delegate.isTerminated(); + } + + @Override + public void shutdownNow() { + synchronized (refreshLock) { + delegate.shutdownNow(); + } + } + + @Override + public boolean awaitTermination(long duration, TimeUnit unit) throws InterruptedException { + return delegate.awaitTermination(duration, unit); + } + + @Override + public void close() { + shutdown(); + } +} diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/HttpJsonCallContextTest.java b/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/HttpJsonCallContextTest.java index 08044522e729..6963ca2c1f88 100644 --- a/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/HttpJsonCallContextTest.java +++ b/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/HttpJsonCallContextTest.java @@ -47,6 +47,7 @@ import com.google.auth.Credentials; import com.google.common.collect.ImmutableMap; import com.google.common.truth.Truth; +import java.time.Duration; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -207,6 +208,19 @@ void testWithLongerTimeout() { .isEqualTo(java.time.Duration.ofSeconds(5)); } + @Test + void testWithNullOrZeroTimeoutClearsExistingTimeout() { + HttpJsonCallContext ctxWithTimeout = + HttpJsonCallContext.createDefault().withTimeoutDuration(Duration.ofSeconds(5)); + + // Sanity check + Truth.assertThat(ctxWithTimeout.getTimeoutDuration()).isEqualTo(Duration.ofSeconds(5)); + + Duration nullTimeout = null; + assertNull(ctxWithTimeout.withTimeoutDuration(nullTimeout).getTimeoutDuration()); + assertNull(ctxWithTimeout.withTimeoutDuration(Duration.ZERO).getTimeoutDuration()); + } + @Test void testMergeWithNullTimeout() { java.time.Duration timeout = java.time.Duration.ofSeconds(10); @@ -334,4 +348,55 @@ void testMergeOptions() { assertEquals(testContext2, mergedContext.getOption(contextKey2)); assertEquals(testContext3, mergedContext.getOption(contextKey3)); } + + @Test + void testWithChannelClearsStaleTransportChannel() { + ManagedHttpJsonChannel channel1 = + mock(ManagedHttpJsonChannel.class, Mockito.withSettings().withoutAnnotations()); + ManagedHttpJsonChannel channel2 = + mock(ManagedHttpJsonChannel.class, Mockito.withSettings().withoutAnnotations()); + + HttpJsonTransportChannel transportChannel1 = + HttpJsonTransportChannel.newBuilder().setManagedChannel(channel1).build(); + + HttpJsonCallContext context = + HttpJsonCallContext.createDefault().withTransportChannel(transportChannel1); + Truth.assertThat(context.getTransportChannel()).isSameInstanceAs(transportChannel1); + + // Retains transportChannel when setting same channel + Truth.assertThat(context.withChannel(channel1).getTransportChannel()) + .isSameInstanceAs(transportChannel1); + + // Clears transportChannel to null when setting null or a different channel + HttpJsonCallContext nullChannelContext = context.withChannel(null); + Truth.assertThat(nullChannelContext.getChannel()).isNull(); + Truth.assertThat(nullChannelContext.getTransportChannel()).isNull(); + Truth.assertThat(context.withChannel(channel2).getTransportChannel()).isNull(); + + // Merging a context with a cleared channel into the original context keeps the original + // channel and transportChannel + HttpJsonCallContext mergedWithNullChannel = context.merge(nullChannelContext); + Truth.assertThat(mergedWithNullChannel.getChannel()).isSameInstanceAs(channel1); + Truth.assertThat(mergedWithNullChannel.getTransportChannel()) + .isSameInstanceAs(transportChannel1); + } + + @Test + void testMergeClearsStaleTransportChannel() { + ManagedHttpJsonChannel channel1 = + mock(ManagedHttpJsonChannel.class, Mockito.withSettings().withoutAnnotations()); + ManagedHttpJsonChannel channel2 = + mock(ManagedHttpJsonChannel.class, Mockito.withSettings().withoutAnnotations()); + + HttpJsonTransportChannel transportChannel1 = + HttpJsonTransportChannel.newBuilder().setManagedChannel(channel1).build(); + + HttpJsonCallContext context1 = + HttpJsonCallContext.createDefault().withTransportChannel(transportChannel1); + HttpJsonCallContext context2 = HttpJsonCallContext.createDefault().withChannel(channel2); + + HttpJsonCallContext merged = context1.merge(context2); + Truth.assertThat(merged.getChannel()).isSameInstanceAs(channel2); + Truth.assertThat(merged.getTransportChannel()).isNull(); + } } diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProviderTest.java b/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProviderTest.java index 8c95c1d2e1c4..74a578da24cb 100644 --- a/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProviderTest.java +++ b/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProviderTest.java @@ -31,21 +31,34 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import com.google.api.client.http.HttpTransport; import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.gax.rpc.HeaderProvider; import com.google.api.gax.rpc.TransportChannelProvider; import com.google.api.gax.rpc.mtls.AbstractMtlsTransportChannelTest; import com.google.api.gax.rpc.mtls.CertificateBasedAccess; +import com.google.api.gax.rpc.testing.FakeMtlsProvider; +import com.google.auth.mtls.CertificateSourceUnavailableException; import com.google.auth.mtls.MtlsProvider; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.GeneralSecurityException; +import java.security.KeyStore; import java.util.Collections; import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; class InstantiatingHttpJsonChannelProviderTest extends AbstractMtlsTransportChannelTest { @@ -55,9 +68,10 @@ class InstantiatingHttpJsonChannelProviderTest extends AbstractMtlsTransportChan @BeforeEach public void setup() throws IOException { - certificateBasedAccess = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "never" : "false"); + certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.getMtlsEndpointUsagePolicy()) + .thenReturn(CertificateBasedAccess.MtlsEndpointUsagePolicy.NEVER); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(false); } @Test @@ -179,20 +193,384 @@ void managedChannelUsesCustomExecutor() throws IOException { instantiatingHttpJsonChannelProvider.getTransportChannel().shutdownNow(); } - @Override - protected Object getMtlsObjectFromTransportChannel( - MtlsProvider provider, CertificateBasedAccess certificateBasedAccess) + @Test + void managedChannelDoesNotShutdownCustomHttpTransport() throws IOException { + HttpTransport mockHttpTransport = Mockito.mock(HttpTransport.class); + + InstantiatingHttpJsonChannelProvider provider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setHttpTransport(mockHttpTransport) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + provider = (InstantiatingHttpJsonChannelProvider) provider.withHeaders(DEFAULT_HEADER_MAP); + + HttpJsonTransportChannel httpJsonTransportChannel = provider.getTransportChannel(); + + // Verify custom transport is injected (direct ManagedHttpJsonChannel when workloadCertPath is + // null) + ManagedHttpJsonInterceptorChannel interceptorChannel = + (ManagedHttpJsonInterceptorChannel) httpJsonTransportChannel.getManagedChannel(); + ManagedHttpJsonInterceptorChannel managedHttpJsonChannel = + (ManagedHttpJsonInterceptorChannel) interceptorChannel.getChannel(); + ManagedHttpJsonChannel channel = managedHttpJsonChannel.getChannel(); + + assertThat(channel.getHttpTransport()).isEqualTo(mockHttpTransport); + + // Perform a shutdown + provider.getTransportChannel().shutdownNow(); + + // Verify that shutdown() was NOT called on the custom HttpTransport + Mockito.verify(mockHttpTransport, Mockito.never()).shutdown(); + } + + @Test + void channelCreation_withWorkloadCertPath_wrapsWithRefreshingHttpJsonChannel() throws IOException, GeneralSecurityException { + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + Mockito.when(certificateBasedAccess.getWorkloadCertPath()).thenReturn("fake/cert/path.json"); + MtlsProvider mtlsProvider = + new FakeMtlsProvider(FakeMtlsProvider.createTestMtlsKeyStore(), "", false); + + InstantiatingHttpJsonChannelProvider provider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setMtlsProvider(mtlsProvider) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + provider = (InstantiatingHttpJsonChannelProvider) provider.withHeaders(DEFAULT_HEADER_MAP); + + HttpJsonTransportChannel httpJsonTransportChannel = provider.getTransportChannel(); + + ManagedHttpJsonInterceptorChannel interceptorChannel = + (ManagedHttpJsonInterceptorChannel) httpJsonTransportChannel.getManagedChannel(); + ManagedHttpJsonInterceptorChannel managedHttpJsonChannel = + (ManagedHttpJsonInterceptorChannel) interceptorChannel.getChannel(); + assertThat(managedHttpJsonChannel.getChannel()).isInstanceOf(RefreshingHttpJsonChannel.class); + + provider.getTransportChannel().shutdownNow(); + } + + @Test + void channelCreation_withCustomHttpTransport_ignoresWorkloadCertPathAndDoesNotWrap() + throws IOException, GeneralSecurityException { + // mTLS is otherwise fully configured, so only the custom transport prevents wrapping. Lenient + // because a custom transport is expected to skip these lookups. + Mockito.lenient().when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + Mockito.lenient() + .when(certificateBasedAccess.getWorkloadCertPath()) + .thenReturn("fake/cert/path.json"); + MtlsProvider mtlsProvider = + new FakeMtlsProvider(FakeMtlsProvider.createTestMtlsKeyStore(), "", false); + HttpTransport mockHttpTransport = Mockito.mock(HttpTransport.class); + + InstantiatingHttpJsonChannelProvider provider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setHttpTransport(mockHttpTransport) + .setMtlsProvider(mtlsProvider) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + provider = (InstantiatingHttpJsonChannelProvider) provider.withHeaders(DEFAULT_HEADER_MAP); + + HttpJsonTransportChannel httpJsonTransportChannel = provider.getTransportChannel(); + + ManagedHttpJsonInterceptorChannel interceptorChannel = + (ManagedHttpJsonInterceptorChannel) httpJsonTransportChannel.getManagedChannel(); + ManagedHttpJsonInterceptorChannel managedHttpJsonChannel = + (ManagedHttpJsonInterceptorChannel) interceptorChannel.getChannel(); + assertThat(managedHttpJsonChannel.getChannel()) + .isNotInstanceOf(RefreshingHttpJsonChannel.class); + Mockito.verify(certificateBasedAccess, Mockito.never()).getWorkloadCertPath(); + + httpJsonTransportChannel.shutdownNow(); + } + + @Test + void getTransportChannel_withMtlsKeyStore_usesMtlsTransport() throws IOException { + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + MtlsProvider mtlsProvider = + new FakeMtlsProvider(FakeMtlsProvider.createTestMtlsKeyStore(), "", false); + + InstantiatingHttpJsonChannelProvider provider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setMtlsProvider(mtlsProvider) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + provider = (InstantiatingHttpJsonChannelProvider) provider.withHeaders(DEFAULT_HEADER_MAP); + + HttpJsonTransportChannel httpJsonTransportChannel = provider.getTransportChannel(); + ManagedHttpJsonInterceptorChannel interceptorChannel = + (ManagedHttpJsonInterceptorChannel) httpJsonTransportChannel.getManagedChannel(); + ManagedHttpJsonChannel channel = + ((ManagedHttpJsonInterceptorChannel) interceptorChannel.getChannel()).getChannel(); + + assertThat(channel).isNotInstanceOf(RefreshingHttpJsonChannel.class); + assertThat(((NetHttpTransport) channel.getHttpTransport()).isMtls()).isTrue(); + + httpJsonTransportChannel.shutdownNow(); + } + + @Test + void refresh_whenKeyStoreUnavailableDuringRotation_keepsCurrentTransport(@TempDir Path tempDir) + throws IOException { + Path certPath = tempDir.resolve("cert.pem"); + Files.write(certPath, "certificate-v1".getBytes(StandardCharsets.UTF_8)); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + Mockito.when(certificateBasedAccess.getWorkloadCertPath()).thenReturn(certPath.toString()); + KeyStore keyStore = FakeMtlsProvider.createTestMtlsKeyStore(); + AtomicReference currentKeyStore = new AtomicReference<>(keyStore); + MtlsProvider mtlsProvider = + new MtlsProvider() { + @Override + public KeyStore getKeyStore() { + return currentKeyStore.get(); + } + + @Override + public boolean isAvailable() { + return true; + } + }; + + InstantiatingHttpJsonChannelProvider provider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setMtlsProvider(mtlsProvider) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + provider = (InstantiatingHttpJsonChannelProvider) provider.withHeaders(DEFAULT_HEADER_MAP); + HttpJsonTransportChannel httpJsonTransportChannel = provider.getTransportChannel(); + ManagedHttpJsonInterceptorChannel interceptorChannel = + (ManagedHttpJsonInterceptorChannel) httpJsonTransportChannel.getManagedChannel(); + RefreshingHttpJsonChannel channel = + (RefreshingHttpJsonChannel) + ((ManagedHttpJsonInterceptorChannel) interceptorChannel.getChannel()).getChannel(); + HttpTransport initialTransport = channel.getHttpTransport(); + assertThat(((NetHttpTransport) initialTransport).isMtls()).isTrue(); + + // The certificate rotates on disk, but the provider cannot return the new key store yet. + Files.write(certPath, "certificate-v2".getBytes(StandardCharsets.UTF_8)); + currentKeyStore.set(null); + assertThat(channel.shouldRefresh()).isTrue(); + channel.refresh(); + + // The current authenticated transport is kept instead of one without a client certificate. + assertThat(channel.getHttpTransport()).isSameInstanceAs(initialTransport); + assertThat(channel.getGeneration()).isEqualTo(0); + assertThat(channel.shouldRefresh()).isTrue(); + + // Once the key store is available again, the next refresh swaps in a new mTLS transport. + currentKeyStore.set(keyStore); + channel.refresh(); + assertThat(channel.getHttpTransport()).isNotSameInstanceAs(initialTransport); + assertThat(((NetHttpTransport) channel.getHttpTransport()).isMtls()).isTrue(); + assertThat(channel.getGeneration()).isEqualTo(1); + + httpJsonTransportChannel.shutdownNow(); + } + + @Test + void getTransportChannel_whenMtlsKeyStoreThrowsIOException_throwsCheckedIOException() + throws Exception { + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + Mockito.when(certificateBasedAccess.getWorkloadCertPath()).thenReturn("fake/cert/path.json"); + MtlsProvider failingMtlsProvider = + Mockito.mock(MtlsProvider.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(failingMtlsProvider.getKeyStore()) + .thenThrow(new IOException("Simulated keystore read failure")); + + InstantiatingHttpJsonChannelProvider provider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setMtlsProvider(failingMtlsProvider) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + final InstantiatingHttpJsonChannelProvider finalProvider = + (InstantiatingHttpJsonChannelProvider) provider.withHeaders(DEFAULT_HEADER_MAP); + + // Must throw checked IOException directly (not wrapped in RuntimeException) + IOException thrown = assertThrows(IOException.class, finalProvider::getTransportChannel); + assertThat(thrown).hasMessageThat().contains("Simulated keystore read failure"); + } + + @Test + void getTransportChannel_whenMtlsActiveAndKeyStoreNull_throwsIOException() { + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + MtlsProvider providerWithNullKeyStore = new FakeMtlsProvider(null, "", false); + + InstantiatingHttpJsonChannelProvider provider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setMtlsProvider(providerWithNullKeyStore) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + InstantiatingHttpJsonChannelProvider finalProvider = + (InstantiatingHttpJsonChannelProvider) provider.withHeaders(DEFAULT_HEADER_MAP); + + IOException thrown = assertThrows(IOException.class, finalProvider::getTransportChannel); + assertThat(thrown).hasMessageThat().contains("Failed to initialize mTLS HttpTransport"); + } + + @Test + void getTransportChannel_whenRotationEnabledAndKeyStoreNull_throwsIOException() { + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + Mockito.when(certificateBasedAccess.getWorkloadCertPath()).thenReturn("fake/cert/path.json"); + MtlsProvider providerWithNullKeyStore = new FakeMtlsProvider(null, "", false); + + InstantiatingHttpJsonChannelProvider provider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setMtlsProvider(providerWithNullKeyStore) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + InstantiatingHttpJsonChannelProvider finalProvider = + (InstantiatingHttpJsonChannelProvider) provider.withHeaders(DEFAULT_HEADER_MAP); + + IOException thrown = assertThrows(IOException.class, finalProvider::getTransportChannel); + assertThat(thrown).hasMessageThat().contains("Failed to initialize mTLS HttpTransport"); + } + + @Test + void getTransportChannel_whenKeyStoreUninitialized_wrapsGeneralSecurityException() + throws Exception { + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + Mockito.when(certificateBasedAccess.getWorkloadCertPath()).thenReturn("fake/cert/path.json"); + // A KeyStore that was never loaded makes transport creation fail with a KeyStoreException. + KeyStore uninitializedKeyStore = KeyStore.getInstance("PKCS12"); + MtlsProvider providerWithBadKeyStore = new FakeMtlsProvider(uninitializedKeyStore, "", false); + + InstantiatingHttpJsonChannelProvider provider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setMtlsProvider(providerWithBadKeyStore) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + InstantiatingHttpJsonChannelProvider finalProvider = + (InstantiatingHttpJsonChannelProvider) provider.withHeaders(DEFAULT_HEADER_MAP); + + // The GeneralSecurityException must be unwrapped from the factory's RuntimeException, so that + // getTransportChannel() surfaces it as the direct cause of its checked IOException. + IOException thrown = assertThrows(IOException.class, finalProvider::getTransportChannel); + assertThat(thrown).hasCauseThat().isInstanceOf(GeneralSecurityException.class); + } + + @Test + void getTransportChannel_whenMtlsKeyStoreThrowsRuntimeException_propagatesUnwrapped() + throws Exception { + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + Mockito.when(certificateBasedAccess.getWorkloadCertPath()).thenReturn("fake/cert/path.json"); + IllegalStateException failure = new IllegalStateException("Simulated keystore failure"); + MtlsProvider failingMtlsProvider = + Mockito.mock(MtlsProvider.class, Mockito.withSettings().withoutAnnotations()); + Mockito.when(failingMtlsProvider.getKeyStore()).thenThrow(failure); + + InstantiatingHttpJsonChannelProvider provider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setMtlsProvider(failingMtlsProvider) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + InstantiatingHttpJsonChannelProvider finalProvider = + (InstantiatingHttpJsonChannelProvider) provider.withHeaders(DEFAULT_HEADER_MAP); + + IllegalStateException thrown = + assertThrows(IllegalStateException.class, finalProvider::getTransportChannel); + assertThat(thrown).isSameInstanceAs(failure); + } + + @Test + void getTransportChannel_whenDefaultMtlsProviderThrowsIOException_propagatesIOException() + throws Exception { + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + IOException expected = new IOException("Corrupt PEM file"); + Mockito.when(certificateBasedAccess.createDefaultMtlsProvider()).thenThrow(expected); + + InstantiatingHttpJsonChannelProvider provider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + InstantiatingHttpJsonChannelProvider finalProvider = + (InstantiatingHttpJsonChannelProvider) provider.withHeaders(DEFAULT_HEADER_MAP); + + IOException thrown = assertThrows(IOException.class, finalProvider::getTransportChannel); + assertThat(thrown).isSameInstanceAs(expected); + } + + @Test + void + getTransportChannel_whenDefaultMtlsProviderThrowsCertificateSourceUnavailable_fallsBackWithoutMtls() + throws Exception { + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + Mockito.when(certificateBasedAccess.createDefaultMtlsProvider()) + .thenThrow(new CertificateSourceUnavailableException("No certificates on disk")); + + InstantiatingHttpJsonChannelProvider provider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + InstantiatingHttpJsonChannelProvider finalProvider = + (InstantiatingHttpJsonChannelProvider) provider.withHeaders(DEFAULT_HEADER_MAP); + + HttpTransport transport = finalProvider.createHttpTransport(); + assertThat(transport).isInstanceOf(NetHttpTransport.class); + assertThat(((NetHttpTransport) transport).isMtls()).isFalse(); + + HttpJsonTransportChannel channel = finalProvider.getTransportChannel(); + assertThat(channel).isNotNull(); + channel.shutdownNow(); + } + + @Test + void createHttpTransport_withMtlsKeyStore_returnsMtlsTransport() + throws IOException, GeneralSecurityException { + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + MtlsProvider provider = + new FakeMtlsProvider(FakeMtlsProvider.createTestMtlsKeyStore(), "", false); + InstantiatingHttpJsonChannelProvider channelProvider = InstantiatingHttpJsonChannelProvider.newBuilder() - .setEndpoint("localhost:8080") + .setEndpoint(DEFAULT_ENDPOINT) .setMtlsProvider(provider) .setCertificateBasedAccess(certificateBasedAccess) - .setHeaderProvider(Collections::emptyMap) - .setExecutor(Runnable::run) .build(); - NetHttpTransport transport = (NetHttpTransport) channelProvider.createHttpTransport(); - return (transport != null && transport.isMtls()) ? transport : null; + + HttpTransport transport = channelProvider.createHttpTransport(); + assertThat(transport).isNotNull(); + assertThat(transport).isInstanceOf(NetHttpTransport.class); + assertThat(((NetHttpTransport) transport).isMtls()).isTrue(); + } + + @Test + void createHttpTransport_whenMtlsProviderNullOrNotUsingClientCert_returnsNonMtlsTransport() + throws IOException, GeneralSecurityException { + InstantiatingHttpJsonChannelProvider nullMtlsProviderChannelProvider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setMtlsProvider(null) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + NetHttpTransport nullMtlsProviderTransport = + (NetHttpTransport) nullMtlsProviderChannelProvider.createHttpTransport(); + assertThat(nullMtlsProviderTransport).isNotNull(); + assertThat(nullMtlsProviderTransport.isMtls()).isFalse(); + + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(false); + MtlsProvider provider = + new FakeMtlsProvider(FakeMtlsProvider.createTestMtlsKeyStore(), "", false); + InstantiatingHttpJsonChannelProvider disabledMtlsChannelProvider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint(DEFAULT_ENDPOINT) + .setMtlsProvider(provider) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + NetHttpTransport disabledMtlsTransport = + (NetHttpTransport) disabledMtlsChannelProvider.createHttpTransport(); + assertThat(disabledMtlsTransport).isNotNull(); + assertThat(disabledMtlsTransport.isMtls()).isFalse(); } @Test @@ -214,4 +592,21 @@ void testConfigureConscryptSecurityProvider_returnsConfiguredBuilder() { HttpJsonConscryptUtils.configureConscryptSecurityProvider(builder); assertThat(result).isSameInstanceAs(builder); } + + @Override + protected Object getMtlsObjectFromTransportChannel( + MtlsProvider provider, CertificateBasedAccess certificateBasedAccess) + throws IOException, GeneralSecurityException { + InstantiatingHttpJsonChannelProvider channelProvider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint("localhost:8080") + .setMtlsProvider(provider) + .setCertificateBasedAccess(certificateBasedAccess) + .setHeaderProvider( + mock(HeaderProvider.class, Mockito.withSettings().withoutAnnotations())) + .setExecutor(mock(Executor.class)) + .build(); + NetHttpTransport transport = (NetHttpTransport) channelProvider.createHttpTransport(); + return (transport != null && transport.isMtls()) ? transport : null; + } } diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProviderTls13Test.java b/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProviderTls13Test.java new file mode 100644 index 000000000000..99bd98bb9fb6 --- /dev/null +++ b/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProviderTls13Test.java @@ -0,0 +1,279 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.httpjson; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.util.SecurityUtils; +import com.google.api.gax.rpc.mtls.CertificateBasedAccess; +import com.google.api.gax.rpc.testing.FakeMtlsProvider; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.InetAddress; +import java.nio.charset.StandardCharsets; +import java.security.KeyStore; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIf; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; + +/** + * Handshake test for the Conscrypt-backed mTLS {@link HttpTransport} built by {@link + * InstantiatingHttpJsonChannelProvider}. + * + *

    Regression test for "Unknown authType: GENERIC": on TLS 1.3, Conscrypt passes the authType + * {@code "GENERIC"} to its trust manager. If the trust manager comes from the JDK (SunJSSE) rather + * than Conscrypt, it rejects server certificates that are issued by a trusted CA and carry a + * KeyUsage extension (as Google front ends' certificates do), and every mTLS HTTP/JSON handshake + * fails. A self-signed server certificate that is itself the trust anchor does not reproduce the + * issue, so the server here presents a CA-issued leaf with KeyUsage. + * + *

    The transport always uses the default trust store, so this test points {@code + * javax.net.ssl.trustStore} at a temporary store containing only the test CA, and restores the + * previous value afterwards. + */ +@EnabledIf( + value = "isConscryptAndTls13Available", + disabledReason = "Conscrypt native library or TLS 1.3 is unavailable on this platform") +class InstantiatingHttpJsonChannelProviderTls13Test { + + private static final String RESOURCE_DIR = "com/google/api/gax/httpjson/"; + private static final String TRUST_STORE_PROPERTY = "javax.net.ssl.trustStore"; + private static final String TRUST_STORE_PASSWORD_PROPERTY = "javax.net.ssl.trustStorePassword"; + private static final String TRUST_STORE_PASSWORD = "changeit"; + + @TempDir File tempDir; + + private String previousTrustStore; + private String previousTrustStorePassword; + private ExecutorService serverExecutor; + + /** Condition for {@link EnabledIf}; must be static because it is used at class level. */ + static boolean isConscryptAndTls13Available() { + return HttpJsonConscryptUtils.getConscryptProvider() != null && isTls13Supported(); + } + + @BeforeEach + void setUp() throws Exception { + previousTrustStore = System.getProperty(TRUST_STORE_PROPERTY); + previousTrustStorePassword = System.getProperty(TRUST_STORE_PASSWORD_PROPERTY); + File trustStoreFile = writeTrustStoreWithTestCa(); + System.setProperty(TRUST_STORE_PROPERTY, trustStoreFile.getAbsolutePath()); + System.setProperty(TRUST_STORE_PASSWORD_PROPERTY, TRUST_STORE_PASSWORD); + + serverExecutor = Executors.newSingleThreadExecutor(); + } + + @AfterEach + void tearDown() { + restoreProperty(TRUST_STORE_PROPERTY, previousTrustStore); + restoreProperty(TRUST_STORE_PASSWORD_PROPERTY, previousTrustStorePassword); + if (serverExecutor != null) { + serverExecutor.shutdownNow(); + } + } + + @Test + void createHttpTransport_withMtlsAndConscrypt_completesTls13HandshakeWithCaIssuedServerCert() + throws Exception { + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + InstantiatingHttpJsonChannelProvider channelProvider = + InstantiatingHttpJsonChannelProvider.newBuilder() + .setEndpoint("localhost:443") + .setMtlsProvider( + new FakeMtlsProvider(FakeMtlsProvider.createTestMtlsKeyStore(), "", false)) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + HttpTransport transport = channelProvider.createHttpTransport(); + assertThat(transport).isNotNull(); + + final SSLServerSocket serverSocket = createTls13ServerSocket(); + try { + Future clientCertificatePresented = + serverExecutor.submit(() -> serveSingleRequest(serverSocket)); + + HttpResponse response = + transport + .createRequestFactory() + .buildGetRequest( + new GenericUrl("https://localhost:" + serverSocket.getLocalPort() + "/")) + .execute(); + try { + assertThat(response.getStatusCode()).isEqualTo(200); + assertThat(response.parseAsString()).isEqualTo("ok"); + } finally { + response.disconnect(); + } + // execute() is synchronous so the server task is already done; timeout guards against hangs. + assertThat(clientCertificatePresented.get(10, TimeUnit.SECONDS)).isTrue(); + } finally { + serverSocket.close(); + } + } + + /** Accepts one connection, answers one HTTP request, and reports if a client cert was sent. */ + private static boolean serveSingleRequest(SSLServerSocket serverSocket) throws Exception { + SSLSocket socket = (SSLSocket) serverSocket.accept(); + try { + socket.startHandshake(); + boolean clientCertificatePresented = socket.getSession().getPeerCertificates().length > 0; + BufferedReader reader = + new BufferedReader( + new InputStreamReader(socket.getInputStream(), StandardCharsets.US_ASCII)); + String line; + while ((line = reader.readLine()) != null && !line.isEmpty()) { + // Drain request headers. + } + OutputStream out = socket.getOutputStream(); + out.write( + ("HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + .getBytes(StandardCharsets.US_ASCII)); + out.flush(); + return clientCertificatePresented; + } finally { + socket.close(); + } + } + + /** + * Creates a JDK TLS 1.3 server that presents a CA-issued leaf certificate with KeyUsage and + * requires (but does not validate) a client certificate. + */ + private static SSLServerSocket createTls13ServerSocket() throws Exception { + KeyStore serverKeyStore; + InputStream certAndKey = openResource("tls13TestServerCertAndKey.pem"); + try { + serverKeyStore = SecurityUtils.createMtlsKeyStore(certAndKey); + } finally { + certAndKey.close(); + } + KeyManagerFactory keyManagerFactory = + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyManagerFactory.init(serverKeyStore, new char[0]); + + SSLContext serverContext = SSLContext.getInstance("TLSv1.3"); + serverContext.init( + keyManagerFactory.getKeyManagers(), new TrustManager[] {new AcceptAllTrustManager()}, null); + SSLServerSocket serverSocket = + (SSLServerSocket) + serverContext + .getServerSocketFactory() + .createServerSocket(0, 1, InetAddress.getLoopbackAddress()); + serverSocket.setEnabledProtocols(new String[] {"TLSv1.3"}); + serverSocket.setNeedClientAuth(true); + return serverSocket; + } + + private File writeTrustStoreWithTestCa() throws Exception { + X509Certificate caCertificate; + InputStream caPem = openResource("tls13TestCa.pem"); + try { + caCertificate = + (X509Certificate) CertificateFactory.getInstance("X.509").generateCertificate(caPem); + } finally { + caPem.close(); + } + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + trustStore.load(null, null); + trustStore.setCertificateEntry("gax-test-ca", caCertificate); + File trustStoreFile = new File(tempDir, "truststore"); + OutputStream out = new FileOutputStream(trustStoreFile); + try { + trustStore.store(out, TRUST_STORE_PASSWORD.toCharArray()); + } finally { + out.close(); + } + return trustStoreFile; + } + + private static InputStream openResource(String name) { + InputStream stream = + InstantiatingHttpJsonChannelProviderTls13Test.class + .getClassLoader() + .getResourceAsStream(RESOURCE_DIR + name); + if (stream == null) { + throw new IllegalStateException("Missing test resource: " + RESOURCE_DIR + name); + } + return stream; + } + + private static boolean isTls13Supported() { + try { + SSLContext.getInstance("TLSv1.3"); + return true; + } catch (Exception e) { + return false; + } + } + + private static void restoreProperty(String key, String value) { + if (value == null) { + System.clearProperty(key); + } else { + System.setProperty(key, value); + } + } + + /** Server-side trust manager: the test only checks that a client certificate is presented. */ + private static final class AcceptAllTrustManager implements X509TrustManager { + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) {} + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) {} + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + } +} diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/RefreshingHttpJsonChannelTest.java b/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/RefreshingHttpJsonChannelTest.java new file mode 100644 index 000000000000..74f462bf831a --- /dev/null +++ b/sdk-platform-java/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/RefreshingHttpJsonChannelTest.java @@ -0,0 +1,607 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.httpjson; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.api.client.http.HttpTransport; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.gax.httpjson.testing.MockHttpService; +import com.google.protobuf.Field; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.function.Supplier; +import org.jspecify.annotations.Nullable; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class RefreshingHttpJsonChannelTest { + private static final ApiMethodDescriptor FAKE_METHOD_DESCRIPTOR = + ApiMethodDescriptor.newBuilder() + .setFullMethodName("google.cloud.v1.Fake/FakeMethod") + .setHttpMethod("POST") + .setRequestFormatter( + ProtoMessageRequestFormatter.newBuilder() + .setPath( + "/fake/v1/name/{name}", + request -> { + Map fields = new HashMap<>(); + ProtoRestSerializer serializer = ProtoRestSerializer.create(); + serializer.putPathParam(fields, "name", request.getName()); + return fields; + }) + .setQueryParamsExtractor(request -> new HashMap<>()) + .setRequestBodyExtractor( + request -> + ProtoRestSerializer.create() + .toBody("*", request.toBuilder().clearName().build(), false)) + .build()) + .setResponseParser( + ProtoMessageResponseParser.newBuilder() + .setDefaultInstance(Field.getDefaultInstance()) + .build()) + .build(); + + private static class FakeHttpJsonClientCall + extends HttpJsonClientCall { + @Override + public void start(Listener responseListener, HttpJsonMetadata requestHeaders) {} + + @Override + public void request(int numMessages) {} + + @Override + public void cancel(@Nullable String message, @Nullable Throwable cause) {} + + @Override + public void sendMessage(RequestT message) {} + + @Override + public void halfClose() {} + } + + private static class FakeManagedHttpJsonChannel extends ManagedHttpJsonChannel { + private volatile boolean isShutdown = false; + private volatile boolean isTerminated = false; + private HttpJsonClientCall nextCall = null; + + @Override + String getEndpoint() { + return "https://fake.endpoint:443"; + } + + @Override + public void shutdown() { + isShutdown = true; + } + + @Override + public void shutdownNow() { + isShutdown = true; + isTerminated = true; + } + + @Override + public boolean isShutdown() { + return isShutdown; + } + + @Override + public boolean isTerminated() { + return isTerminated; + } + + @Override + public boolean awaitTermination(long duration, TimeUnit unit) { + return isTerminated; + } + + @Override + @SuppressWarnings("unchecked") + public HttpJsonClientCall newCall( + ApiMethodDescriptor methodDescriptor, + HttpJsonCallOptions callOptions) { + if (nextCall != null) { + return (HttpJsonClientCall) nextCall; + } + return new FakeHttpJsonClientCall<>(); + } + } + + private AtomicInteger transportFactoryCount; + private HttpTransport lastCreatedTransport; + private FakeManagedHttpJsonChannel lastCreatedChannel; + private String testCertPath; + private String testFingerprint; + private boolean shouldThrowOnFactory; + private List createdChannels; + + private Supplier transportFactory = + () -> { + if (shouldThrowOnFactory) { + throw new RuntimeException("Simulated factory failure"); + } + transportFactoryCount.incrementAndGet(); + lastCreatedTransport = new MockHttpTransport(); + return lastCreatedTransport; + }; + + private Function channelFactory = + transport -> { + lastCreatedChannel = new FakeManagedHttpJsonChannel(); + lastCreatedChannel.setHttpTransport(transport); + return lastCreatedChannel; + }; + + @BeforeEach + void setUp() { + transportFactoryCount = new AtomicInteger(0); + testCertPath = "/fake/path"; + testFingerprint = "fingerprint1"; + shouldThrowOnFactory = false; + createdChannels = new ArrayList<>(); + } + + @AfterEach + void tearDown() { + for (RefreshingHttpJsonChannel channel : createdChannels) { + channel.shutdownNow(); + } + } + + private RefreshingHttpJsonChannel createTestChannel() { + RefreshingHttpJsonChannel ch = + new RefreshingHttpJsonChannel( + () -> transportFactory.get(), + transport -> channelFactory.apply(transport), + "fake/cert/path.json") { + @Override + String getWorkloadCertPath() { + return testCertPath; + } + + @Override + String getCertificateFingerprint(String certPath) { + return testFingerprint; + } + }; + createdChannels.add(ch); + return ch; + } + + private void rotateCertificate(RefreshingHttpJsonChannel channel) { + channel.invalidateDiskFingerprintCache(); + testFingerprint = "fingerprint2"; + } + + @Test + void testShouldRefreshFalseWhenCertPathNull() { + testCertPath = null; + RefreshingHttpJsonChannel channel = createTestChannel(); + assertFalse(channel.shouldRefresh()); + } + + @Test + void testShouldRefreshFalseWhenUnchanged() { + RefreshingHttpJsonChannel channel = createTestChannel(); + + channel.invalidateDiskFingerprintCache(); // Invalidate 1-second cache + assertFalse(channel.shouldRefresh()); + } + + @Test + void testShouldRefreshTrueWhenChanged() { + RefreshingHttpJsonChannel channel = createTestChannel(); + + rotateCertificate(channel); + + assertTrue(channel.shouldRefresh()); + } + + @Test + void shouldRefresh_doesNotCacheNegativeResultAndDetectsSubsequentRotationImmediately() { + RefreshingHttpJsonChannel channel = createTestChannel(); + + // First check returns false (unchanged fingerprint) + assertFalse(channel.shouldRefresh()); + + // Immediately change fingerprint WITHOUT invalidating cache + testFingerprint = "fingerprint2"; + + // Must immediately detect rotation because negative/unchanged checks are not cached for 1s + assertTrue(channel.shouldRefresh()); + + // Refresh updates activeCertFingerprint and clears cache + channel.refresh(); + assertFalse(channel.shouldRefresh()); + } + + @Test + void rotationDuringInitialTransportCreation_isDetectedAndRefreshed() { + transportFactory = + () -> { + if (transportFactoryCount.incrementAndGet() == 1) { + // Simulate the certificate rotating on disk while the initial transport loads it. + testFingerprint = "fingerprint2"; + } + lastCreatedTransport = new MockHttpTransport(); + return lastCreatedTransport; + }; + + RefreshingHttpJsonChannel channel = createTestChannel(); + + // The baseline was recorded before the initial transport was created, so the rotation is seen. + assertTrue(channel.shouldRefresh()); + + channel.refresh(); + assertEquals(2, transportFactoryCount.get()); + assertEquals(1, channel.getGeneration()); + assertFalse(channel.shouldRefresh()); + } + + @Test + void refresh_swapsTransportAndKeepsChannel() { + RefreshingHttpJsonChannel channel = createTestChannel(); + FakeManagedHttpJsonChannel underlyingChannel = lastCreatedChannel; + HttpTransport initialTransport = channel.getHttpTransport(); + assertSame(lastCreatedTransport, initialTransport); + + rotateCertificate(channel); + channel.refresh(); + + assertEquals(2, transportFactoryCount.get()); + assertNotSame(initialTransport, channel.getHttpTransport()); + assertSame(lastCreatedTransport, channel.getHttpTransport()); + // The underlying channel (and its executors) is reused rather than replaced or shut down. + assertSame(underlyingChannel, lastCreatedChannel); + assertFalse(underlyingChannel.isShutdown()); + assertEquals(1, channel.getGeneration()); + assertFalse(channel.shouldRefresh()); + } + + @Test + void refresh_whenCertificateFileEmpty_keepsTransportAndGeneration() { + RefreshingHttpJsonChannel channel = createTestChannel(); + HttpTransport initialTransport = channel.getHttpTransport(); + + // A rotation is detected, but when refresh() re-reads the file the rotator has truncated it + // and not yet written the new certificate. + rotateCertificate(channel); + assertTrue(channel.shouldRefresh()); + testFingerprint = ""; + channel.refresh(); + + // Nothing is swapped and the generation is unchanged, so the failed call is not retried. + assertEquals(1, transportFactoryCount.get()); + assertSame(initialTransport, channel.getHttpTransport()); + assertEquals(0, channel.getGeneration()); + // While the file stays empty, no refresh is requested. + channel.invalidateDiskFingerprintCache(); + assertFalse(channel.shouldRefresh()); + + // Once the new certificate is written, the next check refreshes normally. + testFingerprint = "fingerprint2"; + assertTrue(channel.shouldRefresh()); + channel.refresh(); + assertEquals(2, transportFactoryCount.get()); + assertNotSame(initialTransport, channel.getHttpTransport()); + assertEquals(1, channel.getGeneration()); + } + + @Test + void callCreatedBeforeRefresh_usesOriginalTransport() throws Exception { + MockHttpService originalService = + new MockHttpService(Collections.singletonList(FAKE_METHOD_DESCRIPTOR), "google.com:443"); + MockHttpService rotatedService = + new MockHttpService(Collections.singletonList(FAKE_METHOD_DESCRIPTOR), "google.com:443"); + Field message = Field.newBuilder().setName("bob").setNumber(1).build(); + originalService.addResponse(message); + rotatedService.addResponse(message); + Queue transports = + new ArrayDeque<>(Arrays.asList(originalService, rotatedService)); + transportFactory = transports::remove; + channelFactory = + transport -> + ManagedHttpJsonChannel.newBuilder() + .setEndpoint("google.com:443") + .setHttpTransport(transport) + .build(); + RefreshingHttpJsonChannel channel = createTestChannel(); + HttpJsonCallOptions callOptions = HttpJsonCallOptions.newBuilder().build(); + HttpJsonCallContext callContext = HttpJsonCallContext.createDefault(); + + HttpJsonClientCall callBeforeRefresh = + channel.newCall(FAKE_METHOD_DESCRIPTOR, callOptions); + + rotateCertificate(channel); + channel.refresh(); + assertEquals(1, channel.getGeneration()); + + assertEquals( + message, + HttpJsonClientCalls.futureUnaryCall(callBeforeRefresh, message, callContext) + .get(10, TimeUnit.SECONDS)); + assertEquals(1, originalService.getRequestPaths().size()); + assertEquals(0, rotatedService.getRequestPaths().size()); + + HttpJsonClientCall callAfterRefresh = + channel.newCall(FAKE_METHOD_DESCRIPTOR, callOptions); + assertEquals( + message, + HttpJsonClientCalls.futureUnaryCall(callAfterRefresh, message, callContext) + .get(10, TimeUnit.SECONDS)); + assertEquals(1, originalService.getRequestPaths().size()); + assertEquals(1, rotatedService.getRequestPaths().size()); + } + + @Test + void testRefreshDoesNotCreateTransportWhenShutdown() { + RefreshingHttpJsonChannel channel = createTestChannel(); + assertEquals(1, transportFactoryCount.get()); + + channel.shutdown(); + rotateCertificate(channel); + channel.refresh(); + + assertEquals(1, transportFactoryCount.get()); + assertEquals(0, channel.getGeneration()); + } + + @Test + void shutdown_waitsForInProgressRefresh() throws Exception { + CountDownLatch refreshStarted = new CountDownLatch(1); + CountDownLatch releaseRefresh = new CountDownLatch(1); + transportFactory = + () -> { + if (transportFactoryCount.incrementAndGet() > 1) { + refreshStarted.countDown(); + try { + releaseRefresh.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + return new MockHttpTransport(); + }; + RefreshingHttpJsonChannel channel = createTestChannel(); + FakeManagedHttpJsonChannel underlyingChannel = lastCreatedChannel; + rotateCertificate(channel); + + Thread refreshThread = new Thread(channel::refresh); + Thread shutdownThread = new Thread(channel::shutdown); + try { + refreshThread.start(); + assertTrue(refreshStarted.await(5, TimeUnit.SECONDS)); + shutdownThread.start(); + + shutdownThread.join(200); + assertTrue(shutdownThread.isAlive()); + assertFalse(underlyingChannel.isShutdown()); + } finally { + releaseRefresh.countDown(); + } + refreshThread.join(5000); + shutdownThread.join(5000); + assertFalse(refreshThread.isAlive()); + assertFalse(shutdownThread.isAlive()); + assertEquals(1, channel.getGeneration()); + assertTrue(underlyingChannel.isShutdown()); + } + + @Test + void refresh_concurrentRefreshWhileWaitingForLock_skipsWithoutReadingDisk() throws Exception { + CountDownLatch refreshStarted = new CountDownLatch(1); + CountDownLatch releaseRefresh = new CountDownLatch(1); + transportFactory = + () -> { + if (transportFactoryCount.incrementAndGet() == 2) { + refreshStarted.countDown(); + try { + releaseRefresh.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + return new MockHttpTransport(); + }; + RefreshingHttpJsonChannel channel = createTestChannel(); + rotateCertificate(channel); + + Thread refreshingThread = new Thread(channel::refresh); + Thread waitingThread = new Thread(channel::refresh); + try { + refreshingThread.start(); + assertTrue(refreshStarted.await(5, TimeUnit.SECONDS)); + waitingThread.start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (waitingThread.getState() != Thread.State.BLOCKED) { + assertTrue(System.nanoTime() < deadline); + Thread.sleep(1); + } + // Rotate again before the first refresh completes. A waiter that re-read the certificate + // would see this fingerprint and refresh a second time. + testFingerprint = "fingerprint3"; + } finally { + releaseRefresh.countDown(); + } + refreshingThread.join(5000); + waitingThread.join(5000); + assertFalse(refreshingThread.isAlive()); + assertFalse(waitingThread.isAlive()); + + // Only the initial transport and the concurrent refresh created transports; the waiter skipped. + assertEquals(2, transportFactoryCount.get()); + assertEquals(1, channel.getGeneration()); + // The second rotation is still detected for the next UNAUTHENTICATED failure. + channel.invalidateDiskFingerprintCache(); + assertTrue(channel.shouldRefresh()); + } + + @Test + void testRefreshFactoryExceptionDoesNotWedgeFingerprint() { + RefreshingHttpJsonChannel channel = createTestChannel(); + HttpTransport initialTransport = channel.getHttpTransport(); + assertEquals(1, transportFactoryCount.get()); + + shouldThrowOnFactory = true; + rotateCertificate(channel); + + // Factory failure is logged and the existing transport is kept + channel.refresh(); + assertEquals(1, transportFactoryCount.get()); + assertEquals(0, channel.getGeneration()); + assertSame(initialTransport, channel.getHttpTransport()); + + // Because the factory threw, the new fingerprint is not recorded as active, so the channel + // still reports that it should be refreshed. + assertTrue(channel.shouldRefresh()); + + shouldThrowOnFactory = false; + channel.refresh(); + assertEquals(2, transportFactoryCount.get()); + assertFalse(channel.shouldRefresh()); + } + + @Test + void testShutdownNowSetsIsShutdown() { + RefreshingHttpJsonChannel channel = createTestChannel(); + assertFalse(channel.isShutdown()); + + channel.shutdownNow(); + + assertTrue(channel.isShutdown()); + } + + @Test + void testAwaitTerminationZeroTimeoutOnTerminatedChannelReturnsTrue() throws InterruptedException { + RefreshingHttpJsonChannel channel = createTestChannel(); + lastCreatedChannel.isTerminated = true; + + channel.shutdown(); + assertTrue(channel.awaitTermination(0, TimeUnit.MILLISECONDS)); + } + + @Test + void testChannelDelegationMethods() { + RefreshingHttpJsonChannel channel = createTestChannel(); + FakeManagedHttpJsonChannel underlyingChannel = lastCreatedChannel; + FakeHttpJsonClientCall fakeCall = new FakeHttpJsonClientCall<>(); + underlyingChannel.nextCall = fakeCall; + + assertEquals(underlyingChannel.getEndpoint(), channel.getEndpoint()); + assertEquals(underlyingChannel.getHttpTransport(), channel.getHttpTransport()); + assertEquals(underlyingChannel.getExecutor(), channel.getExecutor()); + assertSame(fakeCall, channel.newCall(null, null)); + } + + @Test + void close_shutsDownUnderlyingChannel() { + RefreshingHttpJsonChannel channel = createTestChannel(); + + channel.close(); + + assertTrue(lastCreatedChannel.isShutdown()); + assertTrue(channel.isShutdown()); + } + + @Test + void testConcurrentNewCallDuringRefresh() throws InterruptedException { + RefreshingHttpJsonChannel channel = createTestChannel(); + int threadCount = 10; + ExecutorService executorService = Executors.newFixedThreadPool(threadCount); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch latch = new CountDownLatch(threadCount); + AtomicInteger successCount = new AtomicInteger(0); + + for (int i = 0; i < threadCount; i++) { + executorService.submit( + () -> { + try { + start.await(); + channel.newCall(null, null); + successCount.incrementAndGet(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + latch.countDown(); + } + }); + } + + rotateCertificate(channel); + // Release the workers just before refreshing so their calls overlap the transport swap. + start.countDown(); + channel.refresh(); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + executorService.shutdown(); + assertTrue(executorService.awaitTermination(5, TimeUnit.SECONDS)); + + assertEquals(threadCount, successCount.get()); + assertEquals(1, channel.getGeneration()); + } + + @Test + void testGenerationIncrementAndLifecycleOnDelegatingWrapper() throws Exception { + RefreshingHttpJsonChannel channel = createTestChannel(); + assertEquals(0, channel.getGeneration()); + + rotateCertificate(channel); + channel.refresh(); + + assertEquals(1, channel.getGeneration()); + + // Verify lifecycle methods on delegating wrapper do not throw NullPointerException + assertFalse(channel.isShutdown()); + assertFalse(channel.isTerminated()); + channel.shutdown(); + assertTrue(channel.isShutdown()); + channel.shutdownNow(); + assertTrue(channel.isTerminated()); + assertTrue(channel.awaitTermination(1, TimeUnit.SECONDS)); + } +} diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/test/resources/com/google/api/gax/httpjson/tls13TestCa.pem b/sdk-platform-java/gax-java/gax-httpjson/src/test/resources/com/google/api/gax/httpjson/tls13TestCa.pem new file mode 100644 index 000000000000..404d562e5297 --- /dev/null +++ b/sdk-platform-java/gax-java/gax-httpjson/src/test/resources/com/google/api/gax/httpjson/tls13TestCa.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDHzCCAgegAwIBAgIUZEyXXbTT3KXPPzN6Z6b30/GQ26wwDQYJKoZIhvcNAQEL +BQAwFjEUMBIGA1UEAwwLR0FYIFRlc3QgQ0EwIBcNMjYwOTMwMDMzNzM5WhgPMjEy +NjA5MDYwMzM3MzlaMBYxFDASBgNVBAMMC0dBWCBUZXN0IENBMIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAkLx+Kh78x1ktPpvGf3ITPS7xwr7rU6Pt/Z34 +AKodCSBCrt/d3BhkGiy8w3IqWN7mIy2ujdoKeUau4NJiYdz3A16XZ1nhvp7J3tK5 +a6SSxO95lVwy70nafTO8ApFYKawKLKNWvGqChM95AI6+Q7dTzzGArpwJxCK0p7dp +4bKjhpSOZaW46VdpCsLwoPJmXNlDpAL5O3nrvdnfilHVCqWHoAZk7ESwQcwm8vP6 +5suLUSvpagPfMRabFTEv91PsmVhB8KjSu8/kGPAC1F7RLyZK31a0Kb5wNwW+69cb +kxjUEhNux7/SHTF250kF2vQ+nFrnUQZSGM8cHU8qpeaOsTFDRwIDAQABo2MwYTAd +BgNVHQ4EFgQU7pHELECvL5hdK8Y3N1x6GZSzSjkwHwYDVR0jBBgwFoAU7pHELECv +L5hdK8Y3N1x6GZSzSjkwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYw +DQYJKoZIhvcNAQELBQADggEBACCyzuu0K4HF0iMu1rj+adlpYUzl6uwkKBuL6fRJ +sTxe0ftgWWcFXJV+P6T+maGu7DMANqbADpWqQbAHMNbSXxPSfGgOd2tS+jg/OOef +go1MhsMUpgxSN9PvWAfhiVIhgF7pdjdVP2qiObx7F/Qo/Xmr0MEy61mmeCbWENBj +jC7DsCo5SWo6kMjvb6dz5G+f+4LElQKCQbBGAOm00XlERE/W7WfUMBzNFhLCHrVs +CRq5lIGxek8DEIC8oXtt/7g0KoKVkOCNj2ZXljEQgi4awoYHqlTRXw/nlctJJ8xs +DLmvfUObHIo66rliDG1Pco/Ce9FVewUJz2WQXJK/mGuuesA= +-----END CERTIFICATE----- diff --git a/sdk-platform-java/gax-java/gax-httpjson/src/test/resources/com/google/api/gax/httpjson/tls13TestServerCertAndKey.pem b/sdk-platform-java/gax-java/gax-httpjson/src/test/resources/com/google/api/gax/httpjson/tls13TestServerCertAndKey.pem new file mode 100644 index 000000000000..b424ecafa2c7 --- /dev/null +++ b/sdk-platform-java/gax-java/gax-httpjson/src/test/resources/com/google/api/gax/httpjson/tls13TestServerCertAndKey.pem @@ -0,0 +1,48 @@ +-----BEGIN CERTIFICATE----- +MIIDTTCCAjWgAwIBAgIUA8zck3uCk3uCK7FqAgRrbQjayGEwDQYJKoZIhvcNAQEL +BQAwFjEUMBIGA1UEAwwLR0FYIFRlc3QgQ0EwIBcNMjYwOTMwMDMzNzM5WhgPMjEy +NjA5MDYwMzM3MzlaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBAMLalJEkZTlTkPbSayD72lYxibZ8JLtRP4OaK/Mr +yvhv8JX5JgSxY3fW2FkpYLBFO3fKFyyPRf2pJTHTpvBjyy/xZnyMHltT9uurF58b +kfAzZDLphJdICTXVL9cEHzCJ+AVFJ85bbLDhCcAd5urCZClqOD6QoM7aLsSurwLQ +eeAP44XwcX+2+VvNfB6YFUgjVreTD+c1xD9OsbMFQuWzv/uDkl6KciOkeuAGOOJh +8ylcugFDOfb9ZWWQRiDo9BiIc/6HIhNk8jo8hCgofcxR/MRZNRKOQ4ilsbNkrjo0 +MusuRQq2mcC4yLZRRo7Jtk2FQYysg2e0mbHKroQhmRIP2wcCAwEAAaOBkjCBjzAa +BgNVHREEEzARgglsb2NhbGhvc3SHBH8AAAEwDgYDVR0PAQH/BAQDAgWgMBMGA1Ud +JQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFL5u77FZtvQz +x06t35j/m9a+PVVNMB8GA1UdIwQYMBaAFO6RxCxAry+YXSvGNzdcehmUs0o5MA0G +CSqGSIb3DQEBCwUAA4IBAQBwAaP5bHnvfObJcl7rKOSfXFKhjm0j5De8BNbS5Dcs +Z22gkuri5qgj8MEwGt4Wu4mxopYD18AbWKt8e7mC5+mJgkydSLvRtJxx5geNw0CD +8TJK8MTpJGJtBKoxC26kxZyHTM6GLO3iAXMzuDFbeW4xpESezoaKsctwlUaAQNO+ +taASUFDy9CJ24SDsiWBmSrTZoNqcCbB3yyYS6Wrx1Et1GYmeTtvx8qo9GjGoil/t +9mWgYhb52Wg+7EO5NQa1eIZ23ZgWg4w7CRqPyjqmOM/ygCmHvoNGuOdbMzSc8o5P +i9S6gTwaU7OBa8vrKdMk+LPkmcHs82qujVYqvU1rMV5a +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDC2pSRJGU5U5D2 +0msg+9pWMYm2fCS7UT+DmivzK8r4b/CV+SYEsWN31thZKWCwRTt3yhcsj0X9qSUx +06bwY8sv8WZ8jB5bU/brqxefG5HwM2Qy6YSXSAk11S/XBB8wifgFRSfOW2yw4QnA +HebqwmQpajg+kKDO2i7Erq8C0HngD+OF8HF/tvlbzXwemBVII1a3kw/nNcQ/TrGz +BULls7/7g5JeinIjpHrgBjjiYfMpXLoBQzn2/WVlkEYg6PQYiHP+hyITZPI6PIQo +KH3MUfzEWTUSjkOIpbGzZK46NDLrLkUKtpnAuMi2UUaOybZNhUGMrINntJmxyq6E +IZkSD9sHAgMBAAECggEAB9NPdCTKC+tXPxJkSwbSVfPXXATQVNWGf3qJmVKA7ElS +mDNRW47kN8PhHHOU9n9RTBZJp/h5Dx7j54Rgh/b6Cgrf2tClx3UvzXcpk7I/wi3O +MF1AffLeq9LMAQ6hUgm4dD26w4bYbqIbCiQpSfRUoXUqnRih3KzWg/+f97T3sA/h +SSNH67sQ/SmaqujmCDsXoVF8Ad0bGzEkEVbn7QgXX+C+DxxA6wGvRER7YRfzhxaz +e1B0rdRLTwlAycKOGPvp+7mVrs1KTELyj8rJp6dNzTpll1W4Y5MeJivSjTpEtPzN +Fo6Wh6+/+wLyXcYUknJMfTlt3GcYdAzhREEXBM34gQKBgQDz0/HtQPulpv8jtoms +m7TiIcRuOHbK+ojt5s9dQ6ikYuCOJ3CnCPLX5T7Eo7yl/NvPhqSr15GbqbHNutLt +FL9Gh5L8H+Z7i/hmmAou59WXIcQOfOS+/ZFi1rpE8Fhp10/dYRSznIG4vaN19DBF +Og4a4n5CR+mRgxcQvt32URiCvwKBgQDMlMJ/rCfN2SAQYXwIE8+7viknMVxyoC1T +ggAVfj+XUdEgPeD8+cKIuQ2E0ino4+wLve6zo5skcw+COKFS7QZcL8xT0oYjRX0I +SZFUGhUnuSDv8rbFx25m27IfQ/FicFFdOHK3n1MbX+I0IfKyYvFrL3DfSuKuZKam +ODIy6PFhuQKBgQCxKxvrDQDNSR6y0HwRE8LHXUnh3N1Ud97vHnsmhXcQm1gXcskE +Vhg+j7CgKWl2ItmFXYxh4O1IcnFjz5Bb/GtP0EsZMATFWNtRkMA1fPdSLZLMBCwi +KxVtACGw3gMgRPcfIN9t8xa/KeyQVLKOWrTNDC458w7gUKR94nJd/GcW+QKBgHiI +HJ/D+h+ZkP6VsNHBel6OhwiEWIdgP9Q3f30qSKUKmz93tZrWO/r5rDFY8UA5KNUy +LfTAq6Mp9zUt0D5fT6P4tVWdb1JQmn3LekgmpdglhnZYdb7I5Q7M2YD6Mrlgm09k +9pGg2QID1+4HIkpfaQVRihrEsBE3U8gSy+CaxbxhAoGBALZn4isHqbH/+H3em7Mc +ITRF2xPhHlCv0SHHn6UHoWVFYXmgF+bjX43wVSjmPL9PusSvH+gBxeYpKQSm9Hjp +eoQirtVJPCVmS5e8pKmmJBAUbRHk1umUR7lwgfTlTnIwEOCD+ApaqWByTdsHNPL1 +fazDXasxJ7y760qDNo7UCbec +-----END PRIVATE KEY----- diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/retrying/RetrySettings.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/retrying/RetrySettings.java index d69fd310c2c0..e6729459b775 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/retrying/RetrySettings.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/retrying/RetrySettings.java @@ -189,7 +189,7 @@ public final org.threeten.bp.Duration getInitialRpcTimeout() { * connection has been terminated). * *

    {@link #getTotalTimeout()} caps how long the logic should keep trying the RPC until it gives - * up completely. If {@link #getTotalTimeout()} is set, initialRpcTimeout should be <= + * up completely. If {@link #getTotalTimeout()} is set, initialRpcTimeout should be <= * totalTimeout. * *

    If there are no configurations, Retries have the default initial RPC timeout value of {@code @@ -356,7 +356,7 @@ public final Builder setInitialRpcTimeout(org.threeten.bp.Duration initialTimeou * the connection has been terminated). * *

    {@link #getTotalTimeout()} caps how long the logic should keep trying the RPC until it - * gives up completely. If {@link #getTotalTimeout()} is set, initialRpcTimeout should be <= + * gives up completely. If {@link #getTotalTimeout()} is set, initialRpcTimeout should be <= * totalTimeout. * *

    If there are no configurations, Retries have the default initial RPC timeout value of @@ -491,7 +491,7 @@ public final org.threeten.bp.Duration getInitialRpcTimeout() { * the connection has been terminated). * *

    {@link #getTotalTimeout()} caps how long the logic should keep trying the RPC until it - * gives up completely. If {@link #getTotalTimeout()} is set, initialRpcTimeout should be <= + * gives up completely. If {@link #getTotalTimeout()} is set, initialRpcTimeout should be <= * totalTimeout. * *

    If there are no configurations, Retries have the default initial RPC timeout value of diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/retrying/StreamingRetryAlgorithm.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/retrying/StreamingRetryAlgorithm.java index e4d5461a6b4e..2ad0f6180e6c 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/retrying/StreamingRetryAlgorithm.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/retrying/StreamingRetryAlgorithm.java @@ -102,13 +102,26 @@ public StreamingRetryAlgorithm( (ServerStreamingAttemptException) previousThrowable; previousThrowable = previousThrowable.getCause(); - // If we have made progress in the last attempt, then reset the delays + // If we have made progress in the last attempt, then reset the delays and attempt counts. + // The next attempt is computed from a fresh baseline, so that result algorithms comparing + // the attempt count with the overall attempt count see the reset stream as a new sequence + // of attempts. The previous overall attempt count is then added back so that it keeps + // increasing across resets. if (attemptException.hasSeenResponses()) { - previousSettings = + int previousOverallAttemptCount = previousSettings.getOverallAttemptCount(); + TimedAttemptSettings resetSettings = createFirstAttempt(context).toBuilder() .setFirstAttemptStartTimeNanos(previousSettings.getFirstAttemptStartTimeNanos()) - .setOverallAttemptCount(previousSettings.getOverallAttemptCount()) .build(); + TimedAttemptSettings nextSettings = + super.createNextAttempt(context, previousThrowable, previousResponse, resetSettings); + if (nextSettings == null) { + return null; + } + return nextSettings.toBuilder() + .setOverallAttemptCount( + nextSettings.getOverallAttemptCount() + previousOverallAttemptCount) + .build(); } } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiCallContext.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiCallContext.java index 2f26f6a6de92..027cc3f6a97e 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiCallContext.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiCallContext.java @@ -41,7 +41,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; /** @@ -54,7 +53,6 @@ * *

    This is transport specific and each transport has an implementation with its own options. */ -@NullMarked @InternalExtensionOnly public interface ApiCallContext extends RetryingContext { @@ -64,6 +62,18 @@ public interface ApiCallContext extends RetryingContext { /** Returns a new ApiCallContext with the given channel set. */ ApiCallContext withTransportChannel(TransportChannel channel); + /** + * Returns the {@link TransportChannel} associated with this call context, or {@code null} if none + * is set. + * + *

    Note: By default, this method returns {@code null}. If an implementation does not override + * this method, automatic mTLS certificate rotation and channel refreshing in retrying callables + * will be disabled. + */ + default TransportChannel getTransportChannel() { + return null; + } + /** Returns a new ApiCallContext with the given Endpoint Context. */ ApiCallContext withEndpointContext(EndpointContext endpointContext); diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiResultRetryAlgorithm.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiResultRetryAlgorithm.java index 7d04d38d2605..f42c52037f63 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiResultRetryAlgorithm.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiResultRetryAlgorithm.java @@ -30,16 +30,68 @@ package com.google.api.gax.rpc; import com.google.api.gax.retrying.BasicResultRetryAlgorithm; +import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.retrying.RetryingContext; +import com.google.api.gax.retrying.TimedAttemptSettings; +import java.time.Duration; import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; /* Package-private for internal use. */ @NullMarked class ApiResultRetryAlgorithm extends BasicResultRetryAlgorithm { - /** Returns true if previousThrowable is an {@link ApiException} that is retryable. */ @Override - public boolean shouldRetry(Throwable previousThrowable, ResponseT previousResponse) { + public @Nullable TimedAttemptSettings createNextAttempt( + @Nullable Throwable previousThrowable, + @Nullable ResponseT previousResponse, + TimedAttemptSettings previousSettings) { + return createNextAttempt(null, previousThrowable, previousResponse, previousSettings); + } + + @Override + public @Nullable TimedAttemptSettings createNextAttempt( + @Nullable RetryingContext context, + @Nullable Throwable previousThrowable, + @Nullable ResponseT previousResponse, + TimedAttemptSettings previousSettings) { + if (isChannelRefreshed(previousThrowable) + && previousSettings.getOverallAttemptCount() == previousSettings.getAttemptCount()) { + RetrySettings globalSettings = previousSettings.getGlobalSettings(); + if (globalSettings.getMaxAttempts() == 0 + && globalSettings.getTotalTimeoutDuration().isZero()) { + globalSettings = globalSettings.toBuilder().setMaxAttempts(1).build(); + } + return previousSettings.toBuilder() + .setGlobalSettings(globalSettings) + .setRetryDelayDuration(Duration.ZERO) + .setRandomizedRetryDelayDuration(Duration.ZERO) + .setAttemptCount(previousSettings.getAttemptCount()) + .setOverallAttemptCount(previousSettings.getOverallAttemptCount() + 1) + .build(); + } + if (isChannelRefreshed(previousThrowable)) { + // The single rotation retry has already been used. Return exhausted settings so the retry + // framework stops, rather than returning null and falling back to exponential backoff. + int exhaustedAttemptCount = previousSettings.getAttemptCount() + 1; + return previousSettings.toBuilder() + .setGlobalSettings( + previousSettings.getGlobalSettings().toBuilder() + .setMaxAttempts(exhaustedAttemptCount) + .build()) + .setAttemptCount(exhaustedAttemptCount) + .setOverallAttemptCount(previousSettings.getOverallAttemptCount() + 1) + .build(); + } + return null; + } + + @Override + public boolean shouldRetry( + @Nullable Throwable previousThrowable, @Nullable ResponseT previousResponse) { + if (isChannelRefreshed(previousThrowable)) { + return true; + } return (previousThrowable instanceof ApiException) && ((ApiException) previousThrowable).isRetryable(); } @@ -52,7 +104,15 @@ public boolean shouldRetry(Throwable previousThrowable, ResponseT previousRespon */ @Override public boolean shouldRetry( - RetryingContext context, Throwable previousThrowable, ResponseT previousResponse) { + RetryingContext context, + @Nullable Throwable previousThrowable, + @Nullable ResponseT previousResponse) { + // A failure on a channel that has since been refreshed (e.g. after an mTLS certificate + // rotation) is eligible for a retry regardless of the configured retryable codes; + // createNextAttempt limits it to a single retry. + if (isChannelRefreshed(previousThrowable)) { + return true; + } if (context.getRetryableCodes() != null) { // Ignore the isRetryable() value of the throwable if the RetryingContext has a specific list // of codes that should be retried. @@ -63,4 +123,9 @@ public boolean shouldRetry( } return shouldRetry(previousThrowable, previousResponse); } + + private static boolean isChannelRefreshed(@Nullable Throwable throwable) { + return throwable instanceof UnauthenticatedException + && ((UnauthenticatedException) throwable).isChannelRefreshed(); + } } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/AttemptCallable.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/AttemptCallable.java index 897e1f2dae8e..148f1851e3b4 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/AttemptCallable.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/AttemptCallable.java @@ -34,7 +34,10 @@ import com.google.api.gax.retrying.NonCancellableFuture; import com.google.api.gax.retrying.RetryingFuture; import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.MoreExecutors; import java.util.concurrent.Callable; +import java.util.logging.Level; +import java.util.logging.Logger; import org.jspecify.annotations.NullMarked; /** @@ -48,6 +51,7 @@ */ @NullMarked class AttemptCallable implements Callable { + private static final Logger LOG = Logger.getLogger(AttemptCallable.class.getName()); private final UnaryCallable callable; private final RequestT request; private final ApiCallContext originalCallContext; @@ -85,8 +89,45 @@ public ResponseT call() { .getTracer() .attemptStarted(request, externalFuture.getAttemptSettings().getOverallAttemptCount()); + TransportChannel transportChannel = callContext.getTransportChannel(); + final long attemptGeneration = + transportChannel != null ? transportChannel.getGeneration() : 0; + ApiFuture internalFuture = callable.futureCall(request, callContext); - externalFuture.setAttemptFuture(internalFuture); + final ApiCallContext finalContext = callContext; + ApiFuture mappedFuture = + ApiFutures.catching( + internalFuture, + UnauthenticatedException.class, + unauthenticatedException -> { + TransportChannel channel = finalContext.getTransportChannel(); + if (channel != null) { + // If another request already refreshed the channel, retry without checking the + // certificate on disk again. Otherwise, check for a rotation and refresh. + boolean shouldRetry = channel.getGeneration() > attemptGeneration; + if (!shouldRetry) { + try { + if (channel.shouldRefresh()) { + channel.refresh(); + } + } catch (Exception e) { + LOG.log( + Level.WARNING, + "Failed to refresh transport channel after authentication error", + e); + } + shouldRetry = channel.getGeneration() > attemptGeneration; + } + + if (shouldRetry) { + throw unauthenticatedException.withChannelRefreshed(); + } + } + throw unauthenticatedException; + }, + MoreExecutors.directExecutor()); + + externalFuture.setAttemptFuture(mappedFuture); } catch (Throwable e) { externalFuture.setAttemptFuture(ApiFutures.immediateFailedFuture(e)); } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/BidiStreamingCallable.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/BidiStreamingCallable.java index 97e22d6ee41f..c8fe6b2a59d3 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/BidiStreamingCallable.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/BidiStreamingCallable.java @@ -29,6 +29,8 @@ */ package com.google.api.gax.rpc; +import java.util.logging.Level; +import java.util.logging.Logger; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; @@ -42,6 +44,7 @@ */ @NullMarked public abstract class BidiStreamingCallable { + private static final Logger LOG = Logger.getLogger(BidiStreamingCallable.class.getName()); protected BidiStreamingCallable() {} @@ -241,11 +244,47 @@ public BidiStreamingCallable withDefaultCallContext( return new BidiStreamingCallable() { @Override public ClientStream internalCall( - ResponseObserver responseObserver, + final ResponseObserver responseObserver, ClientStreamReadyObserver onReady, ApiCallContext thisCallContext) { - return BidiStreamingCallable.this.internalCall( - responseObserver, onReady, defaultCallContext.merge(thisCallContext)); + final ApiCallContext mergedContext = defaultCallContext.merge(thisCallContext); + ResponseObserver refreshingObserver = + new ResponseObserver() { + @Override + public void onStart(StreamController controller) { + responseObserver.onStart(controller); + } + + @Override + public void onResponse(ResponseT response) { + responseObserver.onResponse(response); + } + + @Override + public void onError(Throwable t) { + if (t instanceof UnauthenticatedException) { + TransportChannel transportChannel = mergedContext.getTransportChannel(); + if (transportChannel != null && transportChannel.shouldRefresh()) { + try { + transportChannel.refresh(); + } catch (Exception e) { + LOG.log( + Level.WARNING, + "Failed to refresh transport channel after authentication error", + e); + } + } + } + responseObserver.onError(t); + } + + @Override + public void onComplete() { + responseObserver.onComplete(); + } + }; + + return BidiStreamingCallable.this.internalCall(refreshingObserver, onReady, mergedContext); } }; } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ClientStreamingCallable.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ClientStreamingCallable.java index 854ce3fd5de8..5c87e4c558b7 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ClientStreamingCallable.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ClientStreamingCallable.java @@ -29,6 +29,8 @@ */ package com.google.api.gax.rpc; +import java.util.logging.Level; +import java.util.logging.Logger; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; @@ -42,6 +44,7 @@ */ @NullMarked public abstract class ClientStreamingCallable { + private static final Logger LOG = Logger.getLogger(ClientStreamingCallable.class.getName()); protected ClientStreamingCallable() {} @@ -77,9 +80,40 @@ public ClientStreamingCallable withDefaultCallContext( return new ClientStreamingCallable() { @Override public ApiStreamObserver clientStreamingCall( - ApiStreamObserver responseObserver, ApiCallContext thisCallContext) { - return ClientStreamingCallable.this.clientStreamingCall( - responseObserver, defaultCallContext.merge(thisCallContext)); + final ApiStreamObserver responseObserver, ApiCallContext thisCallContext) { + final ApiCallContext mergedContext = defaultCallContext.merge(thisCallContext); + ApiStreamObserver refreshingObserver = + new ApiStreamObserver() { + @Override + public void onNext(ResponseT response) { + responseObserver.onNext(response); + } + + @Override + public void onError(Throwable t) { + if (t instanceof UnauthenticatedException) { + TransportChannel transportChannel = mergedContext.getTransportChannel(); + if (transportChannel != null && transportChannel.shouldRefresh()) { + try { + transportChannel.refresh(); + } catch (Exception e) { + LOG.log( + Level.WARNING, + "Failed to refresh transport channel after authentication error", + e); + } + } + } + responseObserver.onError(t); + } + + @Override + public void onCompleted() { + responseObserver.onCompleted(); + } + }; + + return ClientStreamingCallable.this.clientStreamingCall(refreshingObserver, mergedContext); } }; } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/EndpointContext.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/EndpointContext.java index 0c0ef9a69b9c..2fdb3974087f 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/EndpointContext.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/EndpointContext.java @@ -34,7 +34,6 @@ import com.google.api.gax.rpc.mtls.CertificateBasedAccess; import com.google.auth.Credentials; import com.google.auth.mtls.CertificateSourceUnavailableException; -import com.google.auth.mtls.DefaultMtlsProviderFactory; import com.google.auth.mtls.MtlsProvider; import com.google.auth.oauth2.ComputeEngineCredentials; import com.google.auto.value.AutoValue; @@ -42,7 +41,6 @@ import com.google.common.base.Strings; import com.google.common.net.HostAndPort; import java.io.IOException; -import java.util.logging.Level; import java.util.logging.Logger; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; @@ -65,7 +63,11 @@ public abstract class EndpointContext { // static block initialization for exception handling static { try { - INSTANCE = EndpointContext.newBuilder().setServiceName("").build(); + INSTANCE = + EndpointContext.newBuilder() + .setServiceName("") + .setCertificateBasedAccess(new CertificateBasedAccess(name -> "false")) + .build(); } catch (IOException e) { throw new RuntimeException("Unable to create a default empty EndpointContext", e); } @@ -309,16 +311,9 @@ private String determineEndpoint() throws IOException { if (certificateBasedAccess.useMtlsClientCertificate()) { if (mtlsProvider == null) { try { - mtlsProvider = DefaultMtlsProviderFactory.create(); + mtlsProvider = certificateBasedAccess.createDefaultMtlsProvider(); } catch (CertificateSourceUnavailableException e) { // This is okay. Leave mtlsProvider as null; - } catch (IOException e) { - LOG.log( - Level.WARNING, - "DefaultMtlsProviderFactory encountered unexpected IOException: " + e.getMessage()); - LOG.log( - Level.WARNING, - "mTLS configuration was detected on the device, but mTLS failed to initialize. Falling back to non-mTLS channel."); } } } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ServerStreamingAttemptCallable.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ServerStreamingAttemptCallable.java index af6e8014e38a..efc374a50ec3 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ServerStreamingAttemptCallable.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/ServerStreamingAttemptCallable.java @@ -37,6 +37,8 @@ import com.google.errorprone.annotations.concurrent.GuardedBy; import java.util.concurrent.Callable; import java.util.concurrent.CancellationException; +import java.util.logging.Level; +import java.util.logging.Logger; import org.jspecify.annotations.NullMarked; /** @@ -96,6 +98,8 @@ */ @NullMarked final class ServerStreamingAttemptCallable implements Callable { + private static final Logger LOG = + Logger.getLogger(ServerStreamingAttemptCallable.class.getName()); private final Object lock = new Object(); private final ServerStreamingCallable innerCallable; @@ -221,6 +225,10 @@ public Void call() { .getTracer() .attemptStarted(request, outerRetryingFuture.getAttemptSettings().getOverallAttemptCount()); + final ApiCallContext finalContext = attemptContext; + TransportChannel channelForAttempt = finalContext.getTransportChannel(); + final long attemptGeneration = + channelForAttempt != null ? channelForAttempt.getGeneration() : 0; innerCallable.call( request, new StateCheckingResponseObserver() { @@ -236,6 +244,35 @@ public void onResponseImpl(ResponseT response) { @Override public void onErrorImpl(Throwable t) { + Throwable cause = t; + if (cause instanceof ServerStreamingAttemptException) { + cause = cause.getCause(); + } + if (cause instanceof UnauthenticatedException) { + UnauthenticatedException unauthenticatedException = (UnauthenticatedException) cause; + TransportChannel transportChannel = finalContext.getTransportChannel(); + if (transportChannel != null) { + // If another request already refreshed the channel, retry without checking the + // certificate on disk again. Otherwise, check for a rotation and refresh. + boolean shouldRetry = transportChannel.getGeneration() > attemptGeneration; + if (!shouldRetry) { + try { + if (transportChannel.shouldRefresh()) { + transportChannel.refresh(); + } + } catch (Exception e) { + LOG.log( + Level.WARNING, + "Failed to refresh transport channel after authentication error", + e); + } + shouldRetry = transportChannel.getGeneration() > attemptGeneration; + } + if (shouldRetry) { + t = unauthenticatedException.withChannelRefreshed(); + } + } + } onAttemptError(t); } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/TransportChannel.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/TransportChannel.java index 1866092e28f9..cad80d4f87b3 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/TransportChannel.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/TransportChannel.java @@ -31,10 +31,8 @@ import com.google.api.core.InternalExtensionOnly; import com.google.api.gax.core.BackgroundResource; -import org.jspecify.annotations.NullMarked; /** Class whose instances can issue RPCs on a particular transport. */ -@NullMarked @InternalExtensionOnly public interface TransportChannel extends BackgroundResource { @@ -49,4 +47,28 @@ public interface TransportChannel extends BackgroundResource { * Returns an empty {@link ApiCallContext} that is compatible with this {@code TransportChannel}. */ ApiCallContext getEmptyCallContext(); + + /** + * Refreshes or recreates the underlying network connections of this transport channel. + * + *

    By default, this is a no-op for transports that do not require stateful connection lifecycle + * management. + */ + default void refresh() {} + + /** + * Returns true if a certificate rotation has been detected on disk and the transport channel + * should be refreshed, or false otherwise. + */ + default boolean shouldRefresh() { + return false; + } + + /** + * Returns a monotonic generation counter tracking the number of successful refreshes or channel + * rotations performed by this transport channel. + */ + default long getGeneration() { + return 0; + } } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/UnauthenticatedException.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/UnauthenticatedException.java index 0c93f07e6b51..d18989264572 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/UnauthenticatedException.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/UnauthenticatedException.java @@ -30,6 +30,7 @@ package com.google.api.gax.rpc; import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; /** * Exception thrown when the request does not have valid authentication credentials for the @@ -37,18 +38,33 @@ */ @NullMarked public class UnauthenticatedException extends ApiException { + // Pinned to the value computed for previous releases (gax 2.83.0 to 2.87.0) so that adding + // members does not break Java serialization compatibility with them. + private static final long serialVersionUID = 6971115068105015909L; + + /** + * Whether this failure happened on a transport channel that has since been refreshed (for + * example, after an mTLS certificate rotation), making the request eligible for a single + * immediate retry on the refreshed channel. Only meaningful within the process that observed the + * failure, so it is not serialized. + */ + private final transient boolean channelRefreshed; + public UnauthenticatedException(Throwable cause, StatusCode statusCode, boolean retryable) { super(cause, statusCode, retryable); + this.channelRefreshed = false; } public UnauthenticatedException( String message, Throwable cause, StatusCode statusCode, boolean retryable) { super(message, cause, statusCode, retryable); + this.channelRefreshed = false; } public UnauthenticatedException( Throwable cause, StatusCode statusCode, boolean retryable, ErrorDetails errorDetails) { super(cause, statusCode, retryable, errorDetails); + this.channelRefreshed = false; } public UnauthenticatedException( @@ -58,5 +74,38 @@ public UnauthenticatedException( boolean retryable, ErrorDetails errorDetails) { super(message, cause, statusCode, retryable, errorDetails); + this.channelRefreshed = false; + } + + private UnauthenticatedException( + @Nullable String message, + @Nullable Throwable cause, + StatusCode statusCode, + boolean retryable, + @Nullable ErrorDetails errorDetails, + boolean channelRefreshed) { + super(message, cause, statusCode, retryable, errorDetails); + this.channelRefreshed = channelRefreshed; + } + + /** Returns whether this failure happened on a transport channel that has since been refreshed. */ + boolean isChannelRefreshed() { + return channelRefreshed; + } + + /** + * Returns a copy of this exception marked as having happened on a channel that has since been + * refreshed. The copy keeps the message, cause, status code, {@link #isRetryable()} value, error + * details, stack trace and suppressed exceptions of this exception. + */ + UnauthenticatedException withChannelRefreshed() { + UnauthenticatedException newEx = + new UnauthenticatedException( + getMessage(), getCause(), getStatusCode(), isRetryable(), getErrorDetails(), true); + newEx.setStackTrace(getStackTrace()); + for (Throwable suppressed : getSuppressed()) { + newEx.addSuppressed(suppressed); + } + return newEx; } } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/mtls/CertificateBasedAccess.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/mtls/CertificateBasedAccess.java index 99f2cca9b53d..59a2b674404c 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/mtls/CertificateBasedAccess.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/mtls/CertificateBasedAccess.java @@ -32,21 +32,23 @@ import com.google.api.core.InternalApi; import com.google.api.gax.rpc.internal.EnvironmentProvider; -import org.jspecify.annotations.NullMarked; +import com.google.auth.mtls.DefaultMtlsProviderFactory; +import com.google.auth.mtls.MtlsProvider; +import com.google.auth.mtls.MtlsUtils; +import com.google.auth.oauth2.PropertyProvider; +import java.io.IOException; /** * Utility class for handling certificate-based access configurations. * - *

    This class handles the processing of GOOGLE_API_USE_CLIENT_CERTIFICATE and - * GOOGLE_API_USE_MTLS_ENDPOINT environment variables according to https://google.aip.dev/auth/4114 + *

    This class handles the processing of GOOGLE_API_USE_CLIENT_CERTIFICATE, + * GOOGLE_API_CERTIFICATE_CONFIG, and GOOGLE_API_USE_MTLS_ENDPOINT configurations. */ -@NullMarked @InternalApi public class CertificateBasedAccess { private final EnvironmentProvider envProvider; - /** The EnvironmentProvider mechanism supports env var injection for unit tests. */ public CertificateBasedAccess(EnvironmentProvider envProvider) { this.envProvider = envProvider; } @@ -66,20 +68,40 @@ public enum MtlsEndpointUsagePolicy { ALWAYS; } + private com.google.auth.oauth2.EnvironmentProvider getAuthEnvProvider() { + return name -> envProvider.getenv(name); + } + + private PropertyProvider getAuthPropertyProvider() { + return System::getProperty; + } + /** Returns if mutual TLS client certificate should be used. */ public boolean useMtlsClientCertificate() { - String useClientCertificate = envProvider.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE"); - return "true".equals(useClientCertificate); + return MtlsUtils.useMtlsClientCertificate(getAuthEnvProvider(), getAuthPropertyProvider()); } /** Returns the current mutual TLS endpoint usage policy. */ public MtlsEndpointUsagePolicy getMtlsEndpointUsagePolicy() { String mtlsEndpointUsagePolicy = envProvider.getenv("GOOGLE_API_USE_MTLS_ENDPOINT"); - if ("never".equals(mtlsEndpointUsagePolicy)) { + if ("never".equalsIgnoreCase(mtlsEndpointUsagePolicy)) { return MtlsEndpointUsagePolicy.NEVER; - } else if ("always".equals(mtlsEndpointUsagePolicy)) { + } else if ("always".equalsIgnoreCase(mtlsEndpointUsagePolicy)) { return MtlsEndpointUsagePolicy.ALWAYS; } return MtlsEndpointUsagePolicy.AUTO; } + + /** + * Resolves and returns the path to the mutual TLS client certificate, or null if none should be + * used. + */ + public String getWorkloadCertPath() { + return MtlsUtils.getWorkloadCertPath(getAuthEnvProvider(), getAuthPropertyProvider()); + } + + /** Creates the default {@link MtlsProvider} for the environment. */ + public MtlsProvider createDefaultMtlsProvider() throws IOException { + return DefaultMtlsProviderFactory.create(); + } } diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/mtls/CertificateRotationTracker.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/mtls/CertificateRotationTracker.java new file mode 100644 index 000000000000..8c390ea8d17b --- /dev/null +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/mtls/CertificateRotationTracker.java @@ -0,0 +1,205 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.rpc.mtls; + +import com.google.api.core.InternalApi; +import com.google.common.base.Strings; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; +import java.util.function.Supplier; +import org.jspecify.annotations.Nullable; + +/** + * Thread-safe helper that tracks workload certificate fingerprints on disk to detect mTLS + * certificate rotations for transport channels ({@code ChannelPool} and {@code + * RefreshingHttpJsonChannel}). + * + *

    For internal use only. + */ +@InternalApi +public class CertificateRotationTracker { + + /** + * Duration (1 second) for which a detected certificate rotation (where the disk fingerprint + * differs from {@code activeCertFingerprint}) is cached in memory. + * + *

    When a certificate rotates on disk, many concurrent in-flight RPCs may fail with {@code + * UNAUTHENTICATED} simultaneously and call {@link #shouldRefresh()}. Caching positive rotation + * detections for 1 second coalesces disk reads ({@code Files.readAllBytes}) and SHA-256 hashing + * across concurrent threads to prevent a thundering herd of file I/O, while bounding maximum + * staleness to 1 second. + */ + private static final long POSITIVE_ROTATION_CACHE_TTL_NANOS = TimeUnit.SECONDS.toNanos(1); + + private static final class DiskCheckResult { + final String fingerprint; + final long sequence; + final long timestampNanos; + + DiskCheckResult(String fingerprint, long sequence, long timestampNanos) { + this.fingerprint = fingerprint; + this.sequence = sequence; + this.timestampNanos = timestampNanos; + } + } + + private final Supplier certPathSupplier; + private final Function fingerprintReader; + private volatile String activeCertFingerprint; + private volatile DiskCheckResult lastDiskCheck = null; + private final ReentrantLock diskCheckLock = new ReentrantLock(); + private final AtomicLong diskCheckSequence = new AtomicLong(0); + + /** + * Creates a tracker for a fixed workload certificate path using {@link + * WorkloadCertificateUtils#getCertificateFingerprint(String)}. + */ + public CertificateRotationTracker(@Nullable String workloadCertPath) { + this(() -> workloadCertPath, WorkloadCertificateUtils::getCertificateFingerprint); + } + + /** + * Creates a tracker with custom certificate path and fingerprint suppliers (used by transports + * that expose package-private overrides for testing). + */ + public CertificateRotationTracker( + Supplier certPathSupplier, Function fingerprintReader) { + this.certPathSupplier = certPathSupplier; + this.fingerprintReader = fingerprintReader; + String initialCertPath = certPathSupplier.get(); + this.activeCertFingerprint = + initialCertPath != null + ? Strings.nullToEmpty(fingerprintReader.apply(initialCertPath)) + : ""; + } + + /** + * Returns {@code true} if a workload certificate path is configured, readable, and its current + * SHA-256 fingerprint on disk differs from the active fingerprint. + */ + public boolean shouldRefresh() { + String certPath = certPathSupplier.get(); + if (certPath == null) { + return false; + } + String currentDiskFingerprint = getOrUpdateDiskFingerprint(certPath); + if (currentDiskFingerprint.isEmpty()) { + return false; + } + return !currentDiskFingerprint.equalsIgnoreCase(activeCertFingerprint); + } + + private String getOrUpdateDiskFingerprint(String certPath) { + long seqBeforeLock = diskCheckSequence.get(); + long now = System.nanoTime(); + DiskCheckResult cached = lastDiskCheck; + if (cached != null + && !cached.fingerprint.isEmpty() + && !cached.fingerprint.equalsIgnoreCase(this.activeCertFingerprint) + && (now - cached.timestampNanos < POSITIVE_ROTATION_CACHE_TTL_NANOS)) { + return cached.fingerprint; + } + + diskCheckLock.lock(); + try { + now = System.nanoTime(); + cached = lastDiskCheck; + if (cached != null + && !cached.fingerprint.isEmpty() + && (cached.sequence > seqBeforeLock + || (!cached.fingerprint.equalsIgnoreCase(this.activeCertFingerprint) + && (now - cached.timestampNanos < POSITIVE_ROTATION_CACHE_TTL_NANOS)))) { + return cached.fingerprint; + } + long newSeq = diskCheckSequence.incrementAndGet(); + String fingerprint = Strings.nullToEmpty(fingerprintReader.apply(certPath)); + if (!fingerprint.isEmpty()) { + lastDiskCheck = new DiskCheckResult(fingerprint, newSeq, System.nanoTime()); + } else { + lastDiskCheck = null; + } + return fingerprint; + } finally { + diskCheckLock.unlock(); + } + } + + /** + * Reads the current certificate fingerprint directly from disk (bypassing the 1-second cache), + * returning {@code ""} if no workload certificate path is configured or if the file is currently + * unreadable/empty. + */ + public String readDiskFingerprint() { + String certPath = certPathSupplier.get(); + if (certPath == null) { + return ""; + } + return Strings.nullToEmpty(fingerprintReader.apply(certPath)); + } + + /** + * Returns {@code true} if {@code diskFingerprint} matches the currently active certificate + * fingerprint. + */ + public boolean isAlreadyActive(String diskFingerprint) { + return diskFingerprint != null && diskFingerprint.equalsIgnoreCase(this.activeCertFingerprint); + } + + /** + * Updates the active certificate fingerprint after a successful channel refresh and clears any + * cached disk check result. + * + *

    This holds {@code diskCheckLock} so that a disk check already in progress cannot write its + * (possibly old) result back to the cache after the switch. + */ + public void markRefreshed(String newFingerprint) { + if (newFingerprint != null && !newFingerprint.isEmpty()) { + diskCheckLock.lock(); + try { + this.activeCertFingerprint = newFingerprint; + this.lastDiskCheck = null; + } finally { + diskCheckLock.unlock(); + } + } + } + + /** Returns the currently active certificate fingerprint (or {@code ""} if none). */ + public String getActiveCertFingerprint() { + return activeCertFingerprint; + } + + /** Invalidates the cached disk check result. Visible for testing. */ + public void invalidateCache() { + this.lastDiskCheck = null; + } +} diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/mtls/WorkloadCertificateUtils.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/mtls/WorkloadCertificateUtils.java new file mode 100644 index 000000000000..94fb36db1ac3 --- /dev/null +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/rpc/mtls/WorkloadCertificateUtils.java @@ -0,0 +1,64 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.rpc.mtls; + +import com.google.api.core.InternalApi; +import com.google.auth.mtls.MtlsUtils; +import java.io.File; + +/** Internal utility class for managing dynamic workload certificates. */ +@InternalApi +public class WorkloadCertificateUtils { + + private static final String EMPTY_FILE_SHA256 = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + private WorkloadCertificateUtils() {} + + /** + * Computes the SHA-256 fingerprint of the certificate file at {@code certPath}, returning {@code + * ""} if the path is {@code null}, unreadable, empty (e.g., temporarily truncated to 0 bytes + * mid-write by an external certificate rotator), or hashes to the empty-byte digest. + * + *

    Returning {@code ""} on unreadable or empty files ensures callers ({@code shouldRefresh()} + * and {@code refresh()}) safely skip refreshing during transient mid-write states rather than + * treating an empty digest as a certificate rotation mismatch. + */ + public static String getCertificateFingerprint(String certPath) { + if (certPath == null || new File(certPath).length() == 0) { + return ""; + } + String fingerprint = MtlsUtils.getCertificateFingerprint(certPath); + if (fingerprint == null || EMPTY_FILE_SHA256.equalsIgnoreCase(fingerprint)) { + return ""; + } + return fingerprint; + } +} diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ApiResultRetryAlgorithmTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ApiResultRetryAlgorithmTest.java index 300a0ad30130..1e730d2db1f3 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ApiResultRetryAlgorithmTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ApiResultRetryAlgorithmTest.java @@ -29,14 +29,25 @@ */ package com.google.api.gax.rpc; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.google.api.core.NanoClock; +import com.google.api.gax.retrying.ExponentialRetryAlgorithm; +import com.google.api.gax.retrying.RetryAlgorithm; +import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.retrying.ServerStreamingAttemptException; +import com.google.api.gax.retrying.StreamingRetryAlgorithm; +import com.google.api.gax.retrying.TimedAttemptSettings; import com.google.api.gax.rpc.StatusCode.Code; import com.google.api.gax.rpc.testing.FakeStatusCode; import com.google.common.collect.Sets; +import java.time.Duration; import java.util.Collections; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -111,4 +122,477 @@ void testShouldRetryWithContextWithEmptyRetryableCodes() { ApiResultRetryAlgorithm algorithm = new ApiResultRetryAlgorithm<>(); assertFalse(algorithm.shouldRetry(context, unavailableException, null)); } + + @Test + void testRotationRetryWithNonRetryableSettings_maxAttemptsOne() { + ApiCallContext context = + mock(ApiCallContext.class, Mockito.withSettings().withoutAnnotations()); + when(context.getRetryableCodes()).thenReturn(Collections.emptySet()); + + RetrySettings settings = + RetrySettings.newBuilder() + .setMaxAttempts(1) + .setInitialRpcTimeoutDuration(Duration.ofSeconds(10)) + .setMaxRpcTimeoutDuration(Duration.ofSeconds(10)) + .setTotalTimeoutDuration(Duration.ofSeconds(10)) + .build(); + + ApiResultRetryAlgorithm resultAlgorithm = new ApiResultRetryAlgorithm<>(); + ExponentialRetryAlgorithm timedAlgorithm = + new ExponentialRetryAlgorithm(settings, NanoClock.getDefaultClock()); + RetryAlgorithm retryAlgorithm = new RetryAlgorithm<>(resultAlgorithm, timedAlgorithm); + + TimedAttemptSettings firstAttempt = retryAlgorithm.createFirstAttempt(context); + UnauthenticatedException rotationEx = rotationException(); + + // First rotation failure: grants immediate free retry without incrementing attemptCount + TimedAttemptSettings nextAttempt = + retryAlgorithm.createNextAttempt(context, rotationEx, null, firstAttempt); + assertNotNull(nextAttempt); + assertEquals(Duration.ZERO, nextAttempt.getRetryDelayDuration()); + assertEquals(Duration.ZERO, nextAttempt.getRandomizedRetryDelayDuration()); + assertEquals(0, nextAttempt.getAttemptCount()); + assertEquals(1, nextAttempt.getOverallAttemptCount()); + assertTrue(retryAlgorithm.shouldRetry(context, rotationEx, null, nextAttempt)); + + // Second consecutive failure: overallAttemptCount (1) != attemptCount (0), so no free retry + TimedAttemptSettings thirdAttempt = + retryAlgorithm.createNextAttempt(context, rotationEx, null, nextAttempt); + assertNotNull(thirdAttempt); + assertEquals(1, thirdAttempt.getAttemptCount()); + assertEquals(2, thirdAttempt.getOverallAttemptCount()); + assertFalse(retryAlgorithm.shouldRetry(context, rotationEx, null, thirdAttempt)); + } + + @Test + void testRotationRetryWithNonRetryableSettings_zeroMaxAttemptsZeroTotalTimeout() { + ApiCallContext context = + mock(ApiCallContext.class, Mockito.withSettings().withoutAnnotations()); + when(context.getRetryableCodes()).thenReturn(Collections.emptySet()); + + RetrySettings settings = + RetrySettings.newBuilder().setMaxAttempts(0).setTotalTimeoutDuration(Duration.ZERO).build(); + + ApiResultRetryAlgorithm resultAlgorithm = new ApiResultRetryAlgorithm<>(); + ExponentialRetryAlgorithm timedAlgorithm = + new ExponentialRetryAlgorithm(settings, NanoClock.getDefaultClock()); + RetryAlgorithm retryAlgorithm = new RetryAlgorithm<>(resultAlgorithm, timedAlgorithm); + + TimedAttemptSettings firstAttempt = retryAlgorithm.createFirstAttempt(context); + UnauthenticatedException rotationEx = rotationException(); + + TimedAttemptSettings nextAttempt = + retryAlgorithm.createNextAttempt(context, rotationEx, null, firstAttempt); + assertNotNull(nextAttempt); + assertEquals(1, nextAttempt.getGlobalSettings().getMaxAttempts()); + assertEquals(0, nextAttempt.getAttemptCount()); + assertEquals(1, nextAttempt.getOverallAttemptCount()); + assertTrue(retryAlgorithm.shouldRetry(context, rotationEx, null, nextAttempt)); + + // Subsequent failure is rejected + TimedAttemptSettings thirdAttempt = + retryAlgorithm.createNextAttempt(context, rotationEx, null, nextAttempt); + assertNotNull(thirdAttempt); + assertFalse(retryAlgorithm.shouldRetry(context, rotationEx, null, thirdAttempt)); + } + + @Test + void testRotationRetryAfterTransientErrorPreservesRemainingBudget() { + ApiCallContext context = + mock(ApiCallContext.class, Mockito.withSettings().withoutAnnotations()); + when(context.getRetryableCodes()).thenReturn(Sets.newHashSet(Code.UNAVAILABLE)); + + RetrySettings settings = + RetrySettings.newBuilder() + .setMaxAttempts(3) + .setInitialRetryDelayDuration(Duration.ofMillis(100)) + .setRetryDelayMultiplier(2.0) + .setMaxRetryDelayDuration(Duration.ofSeconds(1)) + .setTotalTimeoutDuration(Duration.ofSeconds(30)) + .build(); + + ApiResultRetryAlgorithm resultAlgorithm = new ApiResultRetryAlgorithm<>(); + ExponentialRetryAlgorithm timedAlgorithm = + new ExponentialRetryAlgorithm(settings, NanoClock.getDefaultClock()); + RetryAlgorithm retryAlgorithm = new RetryAlgorithm<>(resultAlgorithm, timedAlgorithm); + + TimedAttemptSettings attempt0 = retryAlgorithm.createFirstAttempt(context); + ApiException unavailableEx = + new ApiException(null, new FakeStatusCode(Code.UNAVAILABLE), /* retryable= */ true); + UnauthenticatedException rotationEx = rotationException(); + + // Attempt 0 fails with UNAVAILABLE -> normal retry (attemptCount = 1, overallAttemptCount = 1) + TimedAttemptSettings attempt1 = + retryAlgorithm.createNextAttempt(context, unavailableEx, null, attempt0); + assertEquals(1, attempt1.getAttemptCount()); + assertEquals(1, attempt1.getOverallAttemptCount()); + assertTrue(retryAlgorithm.shouldRetry(context, unavailableEx, null, attempt1)); + + // Attempt 1 fails with rotation 401 -> free zero-delay retry (attemptCount = 1, + // overallAttemptCount = 2) + TimedAttemptSettings attempt2 = + retryAlgorithm.createNextAttempt(context, rotationEx, null, attempt1); + assertEquals(Duration.ZERO, attempt2.getRetryDelayDuration()); + assertEquals(1, attempt2.getAttemptCount()); + assertEquals(2, attempt2.getOverallAttemptCount()); + assertTrue(retryAlgorithm.shouldRetry(context, rotationEx, null, attempt2)); + + // Attempt 2 fails with UNAVAILABLE -> normal retry still allowed (attemptCount = 2 < + // maxAttempts = 3) + TimedAttemptSettings attempt3 = + retryAlgorithm.createNextAttempt(context, unavailableEx, null, attempt2); + assertEquals(2, attempt3.getAttemptCount()); + assertEquals(3, attempt3.getOverallAttemptCount()); + assertTrue(retryAlgorithm.shouldRetry(context, unavailableEx, null, attempt3)); + } + + @Test + void testSecondRotationFailureStopsWithTotalTimeoutAndNoMaxAttempts() { + ApiCallContext context = + mock(ApiCallContext.class, Mockito.withSettings().withoutAnnotations()); + when(context.getRetryableCodes()).thenReturn(Collections.emptySet()); + + RetrySettings settings = + RetrySettings.newBuilder() + .setMaxAttempts(0) + .setInitialRetryDelayDuration(Duration.ofMillis(100)) + .setRetryDelayMultiplier(2.0) + .setMaxRetryDelayDuration(Duration.ofSeconds(1)) + .setTotalTimeoutDuration(Duration.ofMinutes(10)) + .build(); + + ApiResultRetryAlgorithm resultAlgorithm = new ApiResultRetryAlgorithm<>(); + ExponentialRetryAlgorithm timedAlgorithm = + new ExponentialRetryAlgorithm(settings, NanoClock.getDefaultClock()); + RetryAlgorithm retryAlgorithm = new RetryAlgorithm<>(resultAlgorithm, timedAlgorithm); + + TimedAttemptSettings firstAttempt = retryAlgorithm.createFirstAttempt(context); + UnauthenticatedException rotationEx = rotationException(); + + TimedAttemptSettings rotationRetry = + retryAlgorithm.createNextAttempt(context, rotationEx, null, firstAttempt); + assertTrue(retryAlgorithm.shouldRetry(context, rotationEx, null, rotationRetry)); + + // A second rotation-marked failure must stop immediately instead of retrying with backoff + // until the total timeout expires. + TimedAttemptSettings afterSecondFailure = + retryAlgorithm.createNextAttempt(context, rotationEx, null, rotationRetry); + assertNotNull(afterSecondFailure); + assertFalse(retryAlgorithm.shouldRetry(context, rotationEx, null, afterSecondFailure)); + } + + @Test + void testSecondRotationFailureStopsDespiteRemainingMaxAttempts() { + ApiCallContext context = + mock(ApiCallContext.class, Mockito.withSettings().withoutAnnotations()); + when(context.getRetryableCodes()).thenReturn(Sets.newHashSet(Code.UNAVAILABLE)); + + RetrySettings settings = + RetrySettings.newBuilder() + .setMaxAttempts(5) + .setInitialRetryDelayDuration(Duration.ofMillis(100)) + .setRetryDelayMultiplier(2.0) + .setMaxRetryDelayDuration(Duration.ofSeconds(1)) + .setTotalTimeoutDuration(Duration.ofMinutes(10)) + .build(); + + ApiResultRetryAlgorithm resultAlgorithm = new ApiResultRetryAlgorithm<>(); + ExponentialRetryAlgorithm timedAlgorithm = + new ExponentialRetryAlgorithm(settings, NanoClock.getDefaultClock()); + RetryAlgorithm retryAlgorithm = new RetryAlgorithm<>(resultAlgorithm, timedAlgorithm); + + TimedAttemptSettings firstAttempt = retryAlgorithm.createFirstAttempt(context); + UnauthenticatedException rotationEx = rotationException(); + + TimedAttemptSettings rotationRetry = + retryAlgorithm.createNextAttempt(context, rotationEx, null, firstAttempt); + assertTrue(retryAlgorithm.shouldRetry(context, rotationEx, null, rotationRetry)); + + // UNAUTHENTICATED is not a retryable code, so a second rotation-marked failure must stop + // instead of consuming the method's remaining attempts. + TimedAttemptSettings afterSecondFailure = + retryAlgorithm.createNextAttempt(context, rotationEx, null, rotationRetry); + assertNotNull(afterSecondFailure); + assertFalse(retryAlgorithm.shouldRetry(context, rotationEx, null, afterSecondFailure)); + } + + @Test + void testStreamRotationRetryAfterProgressNotBlockedByEarlierRetry() { + ApiCallContext context = + mock(ApiCallContext.class, Mockito.withSettings().withoutAnnotations()); + when(context.getRetryableCodes()).thenReturn(Sets.newHashSet(Code.UNAVAILABLE)); + + RetrySettings settings = + RetrySettings.newBuilder() + .setMaxAttempts(5) + .setInitialRetryDelayDuration(Duration.ofMillis(100)) + .setRetryDelayMultiplier(2.0) + .setMaxRetryDelayDuration(Duration.ofSeconds(1)) + .setTotalTimeoutDuration(Duration.ofMinutes(10)) + .build(); + + StreamingRetryAlgorithm retryAlgorithm = + new StreamingRetryAlgorithm<>( + new ApiResultRetryAlgorithm(), + new ExponentialRetryAlgorithm(settings, NanoClock.getDefaultClock())); + ApiException unavailableEx = + new ApiException(null, new FakeStatusCode(Code.UNAVAILABLE), /* retryable= */ true); + UnauthenticatedException rotationEx = rotationException(); + + // The first attempt fails with UNAVAILABLE before receiving any messages: normal retry. + TimedAttemptSettings attempt1 = + retryAlgorithm.createNextAttempt( + context, + new ServerStreamingAttemptException(unavailableEx, true, false), + null, + retryAlgorithm.createFirstAttempt(context)); + assertEquals(1, attempt1.getAttemptCount()); + assertEquals(1, attempt1.getOverallAttemptCount()); + + // The second attempt receives messages and then fails with a rotation error. The earlier + // retry must not prevent the free rotation retry. + ServerStreamingAttemptException rotationAfterProgress = + new ServerStreamingAttemptException(rotationEx, true, true); + TimedAttemptSettings attempt2 = + retryAlgorithm.createNextAttempt(context, rotationAfterProgress, null, attempt1); + assertNotNull(attempt2); + assertEquals(Duration.ZERO, attempt2.getRetryDelayDuration()); + assertEquals(Duration.ZERO, attempt2.getRandomizedRetryDelayDuration()); + assertEquals(0, attempt2.getAttemptCount()); + assertEquals(2, attempt2.getOverallAttemptCount()); + assertTrue(retryAlgorithm.shouldRetry(context, rotationAfterProgress, null, attempt2)); + + // A repeated rotation error without further progress stops the stream. + ServerStreamingAttemptException rotationWithoutProgress = + new ServerStreamingAttemptException(rotationEx, true, false); + TimedAttemptSettings attempt3 = + retryAlgorithm.createNextAttempt(context, rotationWithoutProgress, null, attempt2); + assertNotNull(attempt3); + assertEquals(3, attempt3.getOverallAttemptCount()); + assertFalse(retryAlgorithm.shouldRetry(context, rotationWithoutProgress, null, attempt3)); + } + + @Test + void testStreamProgressResetKeepsOverallAttemptCountIncreasing() { + ApiCallContext context = + mock(ApiCallContext.class, Mockito.withSettings().withoutAnnotations()); + when(context.getRetryableCodes()).thenReturn(Sets.newHashSet(Code.UNAVAILABLE)); + + RetrySettings settings = + RetrySettings.newBuilder() + .setMaxAttempts(5) + .setInitialRetryDelayDuration(Duration.ofMillis(100)) + .setRetryDelayMultiplier(2.0) + .setMaxRetryDelayDuration(Duration.ofSeconds(1)) + .setTotalTimeoutDuration(Duration.ofMinutes(10)) + .build(); + + StreamingRetryAlgorithm retryAlgorithm = + new StreamingRetryAlgorithm<>( + new ApiResultRetryAlgorithm(), + new ExponentialRetryAlgorithm(settings, NanoClock.getDefaultClock())); + ApiException unavailableEx = + new ApiException(null, new FakeStatusCode(Code.UNAVAILABLE), /* retryable= */ true); + ServerStreamingAttemptException unavailableWithoutProgress = + new ServerStreamingAttemptException(unavailableEx, true, false); + ServerStreamingAttemptException unavailableAfterProgress = + new ServerStreamingAttemptException(unavailableEx, true, true); + + TimedAttemptSettings attempt1 = + retryAlgorithm.createNextAttempt( + context, unavailableWithoutProgress, null, retryAlgorithm.createFirstAttempt(context)); + TimedAttemptSettings attempt2 = + retryAlgorithm.createNextAttempt(context, unavailableWithoutProgress, null, attempt1); + assertEquals(2, attempt2.getAttemptCount()); + assertEquals(2, attempt2.getOverallAttemptCount()); + + // Progress resets the attempt count, but the overall attempt count keeps increasing. + TimedAttemptSettings attempt3 = + retryAlgorithm.createNextAttempt(context, unavailableAfterProgress, null, attempt2); + assertEquals(1, attempt3.getAttemptCount()); + assertEquals(3, attempt3.getOverallAttemptCount()); + assertEquals(settings.getInitialRetryDelayDuration(), attempt3.getRetryDelayDuration()); + assertEquals( + attempt2.getFirstAttemptStartTimeNanos(), attempt3.getFirstAttemptStartTimeNanos()); + assertTrue(retryAlgorithm.shouldRetry(context, unavailableAfterProgress, null, attempt3)); + } + + @Test + void testStreamProgressResetReturnsNullWhenNotRetryable() { + ApiCallContext context = + mock(ApiCallContext.class, Mockito.withSettings().withoutAnnotations()); + when(context.getRetryableCodes()).thenReturn(Collections.emptySet()); + + StreamingRetryAlgorithm retryAlgorithm = + new StreamingRetryAlgorithm<>( + new ApiResultRetryAlgorithm(), + new ExponentialRetryAlgorithm( + RetrySettings.newBuilder().setMaxAttempts(5).build(), NanoClock.getDefaultClock())); + ApiException unavailableEx = + new ApiException(null, new FakeStatusCode(Code.UNAVAILABLE), /* retryable= */ true); + + TimedAttemptSettings next = + retryAlgorithm.createNextAttempt( + context, + new ServerStreamingAttemptException(unavailableEx, true, true), + null, + retryAlgorithm.createFirstAttempt(context)); + assertNull(next); + } + + @Test + void testConfiguredUnauthenticated_notFlagged_usesNormalBackoff() { + // No per-call retryable codes: the method's configuration is carried by isRetryable(). + ApiCallContext context = + mock(ApiCallContext.class, Mockito.withSettings().withoutAnnotations()); + when(context.getRetryableCodes()).thenReturn(null); + RetrySettings settings = + RetrySettings.newBuilder() + .setMaxAttempts(5) + .setInitialRetryDelayDuration(Duration.ofMillis(100)) + .setRetryDelayMultiplier(2.0) + .setMaxRetryDelayDuration(Duration.ofSeconds(1)) + .setTotalTimeoutDuration(Duration.ofMinutes(10)) + .build(); + RetryAlgorithm retryAlgorithm = + new RetryAlgorithm<>( + new ApiResultRetryAlgorithm(), + new ExponentialRetryAlgorithm(settings, NanoClock.getDefaultClock())); + // UNAUTHENTICATED configured as retryable, not caused by a channel refresh. + UnauthenticatedException configuredEx = + new UnauthenticatedException( + "Invalid token", null, new FakeStatusCode(Code.UNAUTHENTICATED), /* retryable= */ true); + + TimedAttemptSettings attempt = retryAlgorithm.createFirstAttempt(context); + for (int i = 1; i < 5; i++) { + attempt = retryAlgorithm.createNextAttempt(context, configuredEx, null, attempt); + assertNotNull(attempt); + assertEquals(i, attempt.getAttemptCount()); + assertEquals(i, attempt.getOverallAttemptCount()); + assertTrue(attempt.getRetryDelayDuration().compareTo(Duration.ZERO) > 0); + assertTrue(retryAlgorithm.shouldRetry(context, configuredEx, null, attempt)); + } + // The fifth failure exhausts maxAttempts = 5. + attempt = retryAlgorithm.createNextAttempt(context, configuredEx, null, attempt); + assertFalse(retryAlgorithm.shouldRetry(context, configuredEx, null, attempt)); + } + + @Test + void testContextRetryableCodesExcludeUnauthenticated_notFlagged_notRetried() { + ApiCallContext context = + mock(ApiCallContext.class, Mockito.withSettings().withoutAnnotations()); + when(context.getRetryableCodes()).thenReturn(Sets.newHashSet(Code.UNAVAILABLE)); + // Retryable according to the method's configuration, but the per-call codes exclude it. + UnauthenticatedException configuredEx = + new UnauthenticatedException( + "Invalid token", null, new FakeStatusCode(Code.UNAUTHENTICATED), /* retryable= */ true); + + ApiResultRetryAlgorithm algorithm = new ApiResultRetryAlgorithm<>(); + assertFalse(algorithm.shouldRetry(context, configuredEx, null)); + assertNull( + algorithm.createNextAttempt( + context, + configuredEx, + null, + new ExponentialRetryAlgorithm( + RetrySettings.newBuilder().setMaxAttempts(5).build(), + NanoClock.getDefaultClock()) + .createFirstAttempt())); + } + + @Test + void testFlagged_nullContext_retriesOnce() { + RetrySettings settings = RetrySettings.newBuilder().setMaxAttempts(1).build(); + RetryAlgorithm retryAlgorithm = + new RetryAlgorithm<>( + new ApiResultRetryAlgorithm(), + new ExponentialRetryAlgorithm(settings, NanoClock.getDefaultClock())); + UnauthenticatedException rotationEx = rotationException(); + + TimedAttemptSettings rotationRetry = + retryAlgorithm.createNextAttempt( + null, rotationEx, null, retryAlgorithm.createFirstAttempt(null)); + assertNotNull(rotationRetry); + assertEquals(Duration.ZERO, rotationRetry.getRetryDelayDuration()); + assertEquals(0, rotationRetry.getAttemptCount()); + assertEquals(1, rotationRetry.getOverallAttemptCount()); + assertTrue(retryAlgorithm.shouldRetry(null, rotationEx, null, rotationRetry)); + + TimedAttemptSettings afterSecondFailure = + retryAlgorithm.createNextAttempt(null, rotationEx, null, rotationRetry); + assertFalse(retryAlgorithm.shouldRetry(null, rotationEx, null, afterSecondFailure)); + } + + @Test + void testFlagged_contextWithoutRetryableCodes_retriesOnce() { + ApiCallContext context = + mock(ApiCallContext.class, Mockito.withSettings().withoutAnnotations()); + when(context.getRetryableCodes()).thenReturn(null); + RetrySettings settings = RetrySettings.newBuilder().setMaxAttempts(1).build(); + RetryAlgorithm retryAlgorithm = + new RetryAlgorithm<>( + new ApiResultRetryAlgorithm(), + new ExponentialRetryAlgorithm(settings, NanoClock.getDefaultClock())); + // Not retryable by configuration (isRetryable() == false), but caused by a channel refresh. + UnauthenticatedException rotationEx = rotationException(); + + TimedAttemptSettings rotationRetry = + retryAlgorithm.createNextAttempt( + context, rotationEx, null, retryAlgorithm.createFirstAttempt(context)); + assertNotNull(rotationRetry); + assertEquals(Duration.ZERO, rotationRetry.getRetryDelayDuration()); + assertTrue(retryAlgorithm.shouldRetry(context, rotationEx, null, rotationRetry)); + + TimedAttemptSettings afterSecondFailure = + retryAlgorithm.createNextAttempt(context, rotationEx, null, rotationRetry); + assertFalse(retryAlgorithm.shouldRetry(context, rotationEx, null, afterSecondFailure)); + } + + @Test + void testConfiguredUnauthenticated_afterRotationRetry_continuesWithNormalPolicy() { + ApiCallContext context = + mock(ApiCallContext.class, Mockito.withSettings().withoutAnnotations()); + when(context.getRetryableCodes()).thenReturn(null); + RetrySettings settings = + RetrySettings.newBuilder() + .setMaxAttempts(3) + .setInitialRetryDelayDuration(Duration.ofMillis(100)) + .setRetryDelayMultiplier(2.0) + .setMaxRetryDelayDuration(Duration.ofSeconds(1)) + .setTotalTimeoutDuration(Duration.ofMinutes(10)) + .build(); + RetryAlgorithm retryAlgorithm = + new RetryAlgorithm<>( + new ApiResultRetryAlgorithm(), + new ExponentialRetryAlgorithm(settings, NanoClock.getDefaultClock())); + UnauthenticatedException configuredEx = + new UnauthenticatedException( + "Expired cert", null, new FakeStatusCode(Code.UNAUTHENTICATED), /* retryable= */ true); + UnauthenticatedException rotationEx = configuredEx.withChannelRefreshed(); + + // A rotation failure gets the free zero-delay retry without consuming an attempt. + TimedAttemptSettings rotationRetry = + retryAlgorithm.createNextAttempt( + context, rotationEx, null, retryAlgorithm.createFirstAttempt(context)); + assertEquals(Duration.ZERO, rotationRetry.getRetryDelayDuration()); + assertEquals(0, rotationRetry.getAttemptCount()); + + // A later, unrelated failure on the same channel follows the configured policy with backoff. + TimedAttemptSettings normalRetry = + retryAlgorithm.createNextAttempt(context, configuredEx, null, rotationRetry); + assertNotNull(normalRetry); + assertEquals(1, normalRetry.getAttemptCount()); + assertTrue(normalRetry.getRetryDelayDuration().compareTo(Duration.ZERO) > 0); + assertTrue(retryAlgorithm.shouldRetry(context, configuredEx, null, normalRetry)); + } + + /** + * An UNAUTHENTICATED failure caused by a channel refresh, where UNAUTHENTICATED is not configured + * as retryable (the common case). + */ + private static UnauthenticatedException rotationException() { + return new UnauthenticatedException( + "Expired cert", null, new FakeStatusCode(Code.UNAUTHENTICATED), /* retryable= */ false) + .withChannelRefreshed(); + } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/AttemptCallableTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/AttemptCallableTest.java index 4b5da578862c..a1e148c241b6 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/AttemptCallableTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/AttemptCallableTest.java @@ -30,16 +30,23 @@ package com.google.api.gax.rpc; import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import com.google.api.core.ApiFuture; import com.google.api.core.SettableApiFuture; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.retrying.RetryingFuture; import com.google.api.gax.retrying.TimedAttemptSettings; import com.google.api.gax.rpc.testing.FakeCallContext; +import com.google.api.gax.rpc.testing.FakeChannel; +import com.google.api.gax.rpc.testing.FakeStatusCode; +import com.google.api.gax.rpc.testing.FakeTransportChannel; import com.google.api.gax.tracing.ApiTracer; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -133,4 +140,388 @@ void testRpcTimeoutIsNotErased() { assertThat(capturedCallContext.getValue().getTimeoutDuration()).isEqualTo(callerTimeout); } + + @Test + void testRefreshedUnauthenticated_flaggedAndPreservesContext() { + FakeTransportChannel transportChannel = + FakeTransportChannel.create(new FakeChannel()).setShouldRefresh(true); + ApiCallContext callContext = + FakeCallContext.createDefault().withTransportChannel(transportChannel); + + UnauthenticatedException originalEx = + new UnauthenticatedException( + "Expired cert", + new IllegalStateException("Root cause"), + FakeStatusCode.of(StatusCode.Code.UNAUTHENTICATED), + false); + originalEx.setStackTrace( + new StackTraceElement[] {new StackTraceElement("foo", "bar", "Baz.java", 123)}); + originalEx.addSuppressed(new RuntimeException("Suppressed cause")); + + SettableApiFuture failedFuture = SettableApiFuture.create(); + failedFuture.setException(originalEx); + when(mockInnerCallable.futureCall(Mockito.anyString(), Mockito.any())).thenReturn(failedFuture); + + AttemptCallable callable = + new AttemptCallable<>(mockInnerCallable, "fake-request", callContext); + callable.setExternalFuture(mockExternalFuture); + + callable.call(); + + ArgumentCaptor futureCaptor = ArgumentCaptor.forClass(ApiFuture.class); + Mockito.verify(mockExternalFuture, Mockito.times(2)).setAttemptFuture(futureCaptor.capture()); + + Throwable thrown = null; + try { + futureCaptor.getValue().get(); + } catch (Exception e) { + thrown = e.getCause(); + } + + assertThat(thrown).isInstanceOf(UnauthenticatedException.class); + UnauthenticatedException rethrown = (UnauthenticatedException) thrown; + assertThat(rethrown.isChannelRefreshed()).isTrue(); + // The original retryable setting is preserved. + assertThat(rethrown.isRetryable()).isFalse(); + assertThat(rethrown.getMessage()).isEqualTo(originalEx.getMessage()); + assertThat(rethrown.getStatusCode()).isEqualTo(originalEx.getStatusCode()); + assertThat(rethrown.getCause()).isEqualTo(originalEx.getCause()); + assertThat(rethrown.getStackTrace()).isEqualTo(originalEx.getStackTrace()); + assertThat(rethrown.getSuppressed().length).isEqualTo(1); + assertThat(rethrown.getSuppressed()[0]).isInstanceOf(RuntimeException.class); + } + + @Test + void testSiblingInFlightRequest_channelRotatedInFlight_flaggedWithoutDuplicateRefresh() { + FakeChannel innerChannel = new FakeChannel(); + innerChannel.setGeneration(1); + // Initially shouldRefresh is false because sibling request already completed the refresh + innerChannel.setShouldRefresh(false); + FakeTransportChannel transportChannel = FakeTransportChannel.create(innerChannel); + + ApiCallContext callContext = + FakeCallContext.createDefault().withTransportChannel(transportChannel); + + UnauthenticatedException originalEx = + new UnauthenticatedException( + "Expired cert", null, FakeStatusCode.of(StatusCode.Code.UNAUTHENTICATED), false); + + SettableApiFuture failedFuture = SettableApiFuture.create(); + failedFuture.setException(originalEx); + when(mockInnerCallable.futureCall(Mockito.anyString(), Mockito.any())) + .thenAnswer( + invocation -> { + // While request was in flight, sibling finished rotation and bumped generation to 2 + innerChannel.setGeneration(2); + return failedFuture; + }); + + AttemptCallable callable = + new AttemptCallable<>(mockInnerCallable, "fake-request", callContext); + callable.setExternalFuture(mockExternalFuture); + + callable.call(); + + ArgumentCaptor futureCaptor = ArgumentCaptor.forClass(ApiFuture.class); + Mockito.verify(mockExternalFuture, Mockito.times(2)).setAttemptFuture(futureCaptor.capture()); + + Throwable thrown = null; + try { + futureCaptor.getValue().get(); + } catch (Exception e) { + thrown = e.getCause(); + } + + assertThat(thrown).isInstanceOf(UnauthenticatedException.class); + UnauthenticatedException rethrown = (UnauthenticatedException) thrown; + // Sibling request should be flagged for a retry on the new channel + assertThat(rethrown.isChannelRefreshed()).isTrue(); + // But should NOT have triggered a second refresh call + assertThat(innerChannel.getRefreshCount()).isEqualTo(0); + } + + @Test + void testPermanentUnauthenticatedFailure_sameGeneration_notFlagged() { + FakeChannel innerChannel = new FakeChannel(); + innerChannel.setGeneration(1); + innerChannel.setShouldRefresh(false); + FakeTransportChannel transportChannel = FakeTransportChannel.create(innerChannel); + + ApiCallContext callContext = + FakeCallContext.createDefault().withTransportChannel(transportChannel); + + UnauthenticatedException originalEx = + new UnauthenticatedException( + "Invalid credentials", null, FakeStatusCode.of(StatusCode.Code.UNAUTHENTICATED), false); + + SettableApiFuture failedFuture = SettableApiFuture.create(); + failedFuture.setException(originalEx); + when(mockInnerCallable.futureCall(Mockito.anyString(), Mockito.any())).thenReturn(failedFuture); + + AttemptCallable callable = + new AttemptCallable<>(mockInnerCallable, "fake-request", callContext); + callable.setExternalFuture(mockExternalFuture); + + callable.call(); + + ArgumentCaptor futureCaptor = ArgumentCaptor.forClass(ApiFuture.class); + Mockito.verify(mockExternalFuture, Mockito.times(2)).setAttemptFuture(futureCaptor.capture()); + + Throwable thrown = null; + try { + futureCaptor.getValue().get(); + } catch (Exception e) { + thrown = e.getCause(); + } + + assertThat(thrown).isInstanceOf(UnauthenticatedException.class); + UnauthenticatedException rethrown = (UnauthenticatedException) thrown; + // Genuine permanent error on same generation is not flagged for a rotation retry + assertThat(rethrown.isRetryable()).isFalse(); + assertThat(rethrown.isChannelRefreshed()).isFalse(); + assertThat(innerChannel.getRefreshCount()).isEqualTo(0); + } + + @Test + void testRefreshThrowsException_notFlaggedWhenGenerationUnchanged() { + AtomicInteger refreshCalls = new AtomicInteger(); + FakeChannel fakeChannel = + new FakeChannel() { + @Override + public boolean shouldRefresh() { + return true; + } + + @Override + public void refresh() { + refreshCalls.incrementAndGet(); + throw new RuntimeException("Refresh error"); + } + }; + FakeTransportChannel transportChannel = FakeTransportChannel.create(fakeChannel); + + ApiCallContext callContext = + FakeCallContext.createDefault().withTransportChannel(transportChannel); + + UnauthenticatedException originalEx = + new UnauthenticatedException( + "Expired cert", null, FakeStatusCode.of(StatusCode.Code.UNAUTHENTICATED), false); + + SettableApiFuture failedFuture = SettableApiFuture.create(); + failedFuture.setException(originalEx); + when(mockInnerCallable.futureCall(Mockito.anyString(), Mockito.any())).thenReturn(failedFuture); + + AttemptCallable callable = + new AttemptCallable<>(mockInnerCallable, "fake-request", callContext); + callable.setExternalFuture(mockExternalFuture); + + callable.call(); + + ArgumentCaptor futureCaptor = ArgumentCaptor.forClass(ApiFuture.class); + Mockito.verify(mockExternalFuture, Mockito.times(2)).setAttemptFuture(futureCaptor.capture()); + + Throwable thrown = null; + try { + futureCaptor.getValue().get(); + } catch (Exception e) { + thrown = e.getCause(); + } + + assertThat(thrown).isInstanceOf(UnauthenticatedException.class); + UnauthenticatedException rethrown = (UnauthenticatedException) thrown; + assertThat(rethrown.isRetryable()).isFalse(); + assertThat(rethrown.isChannelRefreshed()).isFalse(); + assertThat(refreshCalls.get()).isEqualTo(1); + } + + @Test + void testRefreshReturnsWithoutAdvancingGeneration_notFlagged() { + FakeChannel fakeChannel = + new FakeChannel() { + @Override + public boolean shouldRefresh() { + return true; + } + + @Override + public void refresh() { + // Simulates refresh() returning early without rotating any channel (e.g. unreadable + // cert) + } + }; + FakeTransportChannel transportChannel = FakeTransportChannel.create(fakeChannel); + + ApiCallContext callContext = + FakeCallContext.createDefault().withTransportChannel(transportChannel); + + UnauthenticatedException originalEx = + new UnauthenticatedException( + "Expired cert", null, FakeStatusCode.of(StatusCode.Code.UNAUTHENTICATED), false); + + SettableApiFuture failedFuture = SettableApiFuture.create(); + failedFuture.setException(originalEx); + when(mockInnerCallable.futureCall(Mockito.anyString(), Mockito.any())).thenReturn(failedFuture); + + AttemptCallable callable = + new AttemptCallable<>(mockInnerCallable, "fake-request", callContext); + callable.setExternalFuture(mockExternalFuture); + + callable.call(); + + ArgumentCaptor futureCaptor = ArgumentCaptor.forClass(ApiFuture.class); + Mockito.verify(mockExternalFuture, Mockito.times(2)).setAttemptFuture(futureCaptor.capture()); + + Throwable thrown = null; + try { + futureCaptor.getValue().get(); + } catch (Exception e) { + thrown = e.getCause(); + } + + assertThat(thrown).isInstanceOf(UnauthenticatedException.class); + UnauthenticatedException rethrown = (UnauthenticatedException) thrown; + assertThat(rethrown.isRetryable()).isFalse(); + assertThat(rethrown.isChannelRefreshed()).isFalse(); + } + + @Test + void testRefreshThrowsException_flaggedIfConcurrentThreadAdvancedGeneration() { + FakeChannel fakeChannel = + new FakeChannel() { + @Override + public boolean shouldRefresh() { + return true; + } + + @Override + public void refresh() { + // Concurrent thread advanced generation before/during refresh failure + setGeneration(getGeneration() + 1); + throw new RuntimeException("Refresh error on this thread"); + } + }; + FakeTransportChannel transportChannel = FakeTransportChannel.create(fakeChannel); + + ApiCallContext callContext = + FakeCallContext.createDefault().withTransportChannel(transportChannel); + + UnauthenticatedException originalEx = + new UnauthenticatedException( + "Expired cert", null, FakeStatusCode.of(StatusCode.Code.UNAUTHENTICATED), false); + + SettableApiFuture failedFuture = SettableApiFuture.create(); + failedFuture.setException(originalEx); + when(mockInnerCallable.futureCall(Mockito.anyString(), Mockito.any())).thenReturn(failedFuture); + + AttemptCallable callable = + new AttemptCallable<>(mockInnerCallable, "fake-request", callContext); + callable.setExternalFuture(mockExternalFuture); + + callable.call(); + + ArgumentCaptor futureCaptor = ArgumentCaptor.forClass(ApiFuture.class); + Mockito.verify(mockExternalFuture, Mockito.times(2)).setAttemptFuture(futureCaptor.capture()); + + Throwable thrown = null; + try { + futureCaptor.getValue().get(); + } catch (Exception e) { + thrown = e.getCause(); + } + + assertThat(thrown).isInstanceOf(UnauthenticatedException.class); + UnauthenticatedException rethrown = (UnauthenticatedException) thrown; + assertThat(rethrown.isChannelRefreshed()).isTrue(); + } + + @Test + void testGenerationAlreadyAdvanced_flaggedWithoutCheckingShouldRefresh() { + AtomicInteger shouldRefreshCalls = new AtomicInteger(); + FakeChannel fakeChannel = + new FakeChannel() { + @Override + public boolean shouldRefresh() { + shouldRefreshCalls.incrementAndGet(); + return true; + } + }; + UnauthenticatedException originalEx = + new UnauthenticatedException( + "Expired cert", null, FakeStatusCode.of(StatusCode.Code.UNAUTHENTICATED), false); + SettableApiFuture failedFuture = SettableApiFuture.create(); + failedFuture.setException(originalEx); + when(mockInnerCallable.futureCall(Mockito.anyString(), Mockito.any())) + .thenAnswer( + invocation -> { + // Another request rotated the channel while this one was in flight. + fakeChannel.setGeneration(fakeChannel.getGeneration() + 1); + return failedFuture; + }); + + UnauthenticatedException rethrown = callAndGetUnauthenticated(fakeChannel); + + assertThat(rethrown.isChannelRefreshed()).isTrue(); + // The certificate on disk is not checked again, and no second refresh happens. + assertThat(shouldRefreshCalls.get()).isEqualTo(0); + assertThat(fakeChannel.getRefreshCount()).isEqualTo(0); + } + + @Test + void testShouldRefreshThrows_originalUnauthenticatedPreserved() { + FakeChannel fakeChannel = + new FakeChannel() { + @Override + public boolean shouldRefresh() { + throw new IllegalStateException("Unable to read certificate"); + } + }; + UnauthenticatedException originalEx = + new UnauthenticatedException( + "Expired cert", null, FakeStatusCode.of(StatusCode.Code.UNAUTHENTICATED), false); + SettableApiFuture failedFuture = SettableApiFuture.create(); + failedFuture.setException(originalEx); + when(mockInnerCallable.futureCall(Mockito.anyString(), Mockito.any())).thenReturn(failedFuture); + + UnauthenticatedException rethrown = callAndGetUnauthenticated(fakeChannel); + + assertThat(rethrown).isSameInstanceAs(originalEx); + assertThat(rethrown.isChannelRefreshed()).isFalse(); + assertThat(fakeChannel.getRefreshCount()).isEqualTo(0); + } + + @Test + void testConfiguredRetryableUnauthenticated_sameGeneration_notFlagged() { + FakeChannel fakeChannel = new FakeChannel().setShouldRefresh(false); + UnauthenticatedException originalEx = + new UnauthenticatedException( + "Token expired", null, FakeStatusCode.of(StatusCode.Code.UNAUTHENTICATED), true); + SettableApiFuture failedFuture = SettableApiFuture.create(); + failedFuture.setException(originalEx); + when(mockInnerCallable.futureCall(Mockito.anyString(), Mockito.any())).thenReturn(failedFuture); + + UnauthenticatedException rethrown = callAndGetUnauthenticated(fakeChannel); + + // Left to the normal retry policy: still retryable, but not a rotation retry. + assertThat(rethrown).isSameInstanceAs(originalEx); + assertThat(rethrown.isRetryable()).isTrue(); + assertThat(rethrown.isChannelRefreshed()).isFalse(); + } + + private UnauthenticatedException callAndGetUnauthenticated(FakeChannel fakeChannel) { + ApiCallContext callContext = + FakeCallContext.createDefault() + .withTransportChannel(FakeTransportChannel.create(fakeChannel)); + AttemptCallable callable = + new AttemptCallable<>(mockInnerCallable, "fake-request", callContext); + callable.setExternalFuture(mockExternalFuture); + + callable.call(); + + ArgumentCaptor futureCaptor = ArgumentCaptor.forClass(ApiFuture.class); + Mockito.verify(mockExternalFuture, Mockito.times(2)).setAttemptFuture(futureCaptor.capture()); + ExecutionException e = + assertThrows(ExecutionException.class, () -> futureCaptor.getValue().get()); + assertThat(e.getCause()).isInstanceOf(UnauthenticatedException.class); + return (UnauthenticatedException) e.getCause(); + } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/EndpointContextTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/EndpointContextTest.java index 4c1b5320de4c..e388676a8c89 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/EndpointContextTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/EndpointContextTest.java @@ -38,6 +38,7 @@ import com.google.api.gax.rpc.mtls.CertificateBasedAccess; import com.google.api.gax.rpc.testing.FakeMtlsProvider; import com.google.auth.Credentials; +import com.google.auth.mtls.CertificateSourceUnavailableException; import com.google.auth.mtls.MtlsProvider; import com.google.auth.oauth2.ComputeEngineCredentials; import com.google.common.truth.Truth; @@ -76,9 +77,10 @@ void mtlsEndpointResolver_switchToMtlsAllowedIsFalse() throws IOException { new FakeMtlsProvider( FakeMtlsProvider.createTestMtlsKeyStore(), "", throwExceptionForGetKeyStore); boolean switchToMtlsEndpointAllowed = false; - CertificateBasedAccess certificateBasedAccess = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "auto" : "true"); + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.getMtlsEndpointUsagePolicy()) + .thenReturn(CertificateBasedAccess.MtlsEndpointUsagePolicy.AUTO); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); Truth.assertThat( defaultEndpointContextBuilder.mtlsEndpointResolver( DEFAULT_ENDPOINT, @@ -96,9 +98,10 @@ void mtlsEndpointResolver_switchToMtlsAllowedIsTrue_mtlsUsageAuto() throws IOExc new FakeMtlsProvider( FakeMtlsProvider.createTestMtlsKeyStore(), "", throwExceptionForGetKeyStore); boolean switchToMtlsEndpointAllowed = true; - CertificateBasedAccess certificateBasedAccess = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "auto" : "true"); + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.getMtlsEndpointUsagePolicy()) + .thenReturn(CertificateBasedAccess.MtlsEndpointUsagePolicy.AUTO); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); Truth.assertThat( defaultEndpointContextBuilder.mtlsEndpointResolver( DEFAULT_ENDPOINT, @@ -116,9 +119,10 @@ void mtlsEndpointResolver_switchToMtlsAllowedIsTrue_mtlsUsageAlways() throws IOE new FakeMtlsProvider( FakeMtlsProvider.createTestMtlsKeyStore(), "", throwExceptionForGetKeyStore); boolean switchToMtlsEndpointAllowed = true; - CertificateBasedAccess certificateBasedAccess = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "always" : "true"); + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.getMtlsEndpointUsagePolicy()) + .thenReturn(CertificateBasedAccess.MtlsEndpointUsagePolicy.ALWAYS); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); Truth.assertThat( defaultEndpointContextBuilder.mtlsEndpointResolver( DEFAULT_ENDPOINT, @@ -136,9 +140,10 @@ void mtlsEndpointResolver_switchToMtlsAllowedIsTrue_mtlsUsageNever() throws IOEx new FakeMtlsProvider( FakeMtlsProvider.createTestMtlsKeyStore(), "", throwExceptionForGetKeyStore); boolean switchToMtlsEndpointAllowed = true; - CertificateBasedAccess certificateBasedAccess = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "never" : "true"); + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.getMtlsEndpointUsagePolicy()) + .thenReturn(CertificateBasedAccess.MtlsEndpointUsagePolicy.NEVER); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); Truth.assertThat( defaultEndpointContextBuilder.mtlsEndpointResolver( DEFAULT_ENDPOINT, @@ -155,9 +160,10 @@ void mtlsEndpointResolver_switchToMtlsAllowedIsTrue_useCertificateIsFalse_nullMt boolean throwExceptionForGetKeyStore = false; MtlsProvider mtlsProvider = new FakeMtlsProvider(null, "", throwExceptionForGetKeyStore); boolean switchToMtlsEndpointAllowed = true; - CertificateBasedAccess certificateBasedAccess = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "auto" : "false"); + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.getMtlsEndpointUsagePolicy()) + .thenReturn(CertificateBasedAccess.MtlsEndpointUsagePolicy.AUTO); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(false); Truth.assertThat( defaultEndpointContextBuilder.mtlsEndpointResolver( DEFAULT_ENDPOINT, @@ -173,9 +179,10 @@ void mtlsEndpointResolver_getKeyStore_throwsIOException() throws IOException { boolean throwExceptionForGetKeyStore = true; MtlsProvider mtlsProvider = new FakeMtlsProvider(null, "", throwExceptionForGetKeyStore); boolean switchToMtlsEndpointAllowed = true; - CertificateBasedAccess certificateBasedAccess = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "auto" : "true"); + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.getMtlsEndpointUsagePolicy()) + .thenReturn(CertificateBasedAccess.MtlsEndpointUsagePolicy.AUTO); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); assertThrows( IOException.class, () -> @@ -271,9 +278,10 @@ void endpointContextBuild_noUniverseDomain_noEndpoints() throws IOException { void endpointContextBuild_mtlsConfigured_GDU() throws IOException { MtlsProvider mtlsProvider = new FakeMtlsProvider(FakeMtlsProvider.createTestMtlsKeyStore(), "", false); - CertificateBasedAccess certificateBasedAccess = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "always" : "true"); + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.getMtlsEndpointUsagePolicy()) + .thenReturn(CertificateBasedAccess.MtlsEndpointUsagePolicy.ALWAYS); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); EndpointContext endpointContext = defaultEndpointContextBuilder .setClientSettingsEndpoint(null) @@ -292,9 +300,10 @@ void endpointContextBuild_mtlsConfigured_nonGDU_throwsIllegalArgumentException() throws IOException { MtlsProvider mtlsProvider = new FakeMtlsProvider(FakeMtlsProvider.createTestMtlsKeyStore(), "", false); - CertificateBasedAccess certificateBasedAccess = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "always" : "true"); + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.getMtlsEndpointUsagePolicy()) + .thenReturn(CertificateBasedAccess.MtlsEndpointUsagePolicy.ALWAYS); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); EndpointContext.Builder endpointContextBuilder = defaultEndpointContextBuilder .setUniverseDomain("random.com") @@ -735,4 +744,39 @@ void getUrlDomain_nonGDUUniverseDomain() throws IOException { defaultEndpointContextBuilder.setUniverseDomain("random.com").build(); Truth.assertThat(endpointContext.getUrlDomain()).isEqualTo("test.random.com"); } + + @Test + void endpointContextBuild_whenDefaultMtlsProviderThrowsIOException_propagatesIOException() + throws Exception { + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + IOException expected = new IOException("Corrupt PEM file"); + Mockito.when(certificateBasedAccess.createDefaultMtlsProvider()).thenThrow(expected); + + EndpointContext.Builder builder = + defaultEndpointContextBuilder + .setMtlsProvider(null) + .setCertificateBasedAccess(certificateBasedAccess); + IOException actual = assertThrows(IOException.class, builder::build); + Truth.assertThat(actual).isSameInstanceAs(expected); + } + + @Test + void + endpointContextBuild_whenDefaultMtlsProviderThrowsCertificateSourceUnavailable_fallsBackToRegularEndpoint() + throws Exception { + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + Mockito.when(certificateBasedAccess.getMtlsEndpointUsagePolicy()) + .thenReturn(CertificateBasedAccess.MtlsEndpointUsagePolicy.AUTO); + Mockito.when(certificateBasedAccess.createDefaultMtlsProvider()) + .thenThrow(new CertificateSourceUnavailableException("No certificates on disk")); + + EndpointContext endpointContext = + defaultEndpointContextBuilder + .setMtlsProvider(null) + .setCertificateBasedAccess(certificateBasedAccess) + .build(); + Truth.assertThat(endpointContext.resolvedEndpoint()).isEqualTo(DEFAULT_ENDPOINT); + } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ServerStreamingAttemptCallableTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ServerStreamingAttemptCallableTest.java index 4cdd3cc018cc..9cfe8ac8df5d 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ServerStreamingAttemptCallableTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/ServerStreamingAttemptCallableTest.java @@ -29,27 +29,39 @@ */ package com.google.api.gax.rpc; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import com.google.api.core.AbstractApiFuture; import com.google.api.core.ApiFuture; +import com.google.api.core.NanoClock; +import com.google.api.gax.core.FakeApiClock; +import com.google.api.gax.retrying.ExponentialRetryAlgorithm; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.retrying.RetryingFuture; import com.google.api.gax.retrying.ServerStreamingAttemptException; +import com.google.api.gax.retrying.SimpleStreamResumptionStrategy; import com.google.api.gax.retrying.StreamResumptionStrategy; +import com.google.api.gax.retrying.StreamingRetryAlgorithm; import com.google.api.gax.retrying.TimedAttemptSettings; import com.google.api.gax.rpc.StatusCode.Code; import com.google.api.gax.rpc.testing.FakeApiException; import com.google.api.gax.rpc.testing.FakeCallContext; +import com.google.api.gax.rpc.testing.FakeChannel; +import com.google.api.gax.rpc.testing.FakeStatusCode; +import com.google.api.gax.rpc.testing.FakeTransportChannel; import com.google.api.gax.rpc.testing.MockStreamingApi.MockServerStreamingCall; import com.google.api.gax.rpc.testing.MockStreamingApi.MockServerStreamingCallable; import com.google.api.gax.tracing.BaseApiTracer; import com.google.common.collect.Queues; import com.google.common.truth.Truth; +import java.time.Duration; import java.util.concurrent.BlockingDeque; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -248,6 +260,188 @@ void testInitialRetry() { Truth.assertThat(call.getRequest()).isEqualTo("request > 0"); } + @Test + @SuppressWarnings("ConstantConditions") + void testUnauthenticatedRefreshWithoutGenerationAdvance_notFlagged() { + TransportChannel transportChannel = Mockito.mock(TransportChannel.class); + Mockito.when(transportChannel.shouldRefresh()).thenReturn(true); + + ApiCallContext context = Mockito.mock(ApiCallContext.class); + Mockito.when(context.getTransportChannel()).thenReturn(transportChannel); + Mockito.when(context.getTracer()).thenReturn(BaseApiTracer.getInstance()); + Mockito.when(context.getTimeoutDuration()).thenReturn(Duration.ofHours(5)); + + resumptionStrategy = new MyStreamResumptionStrategy(); + ServerStreamingAttemptCallable callable = createCallable(context); + callable.start(); + + MockServerStreamingCall call = innerCallable.popLastCall(); + + // Send initial error + UnauthenticatedException initialError = + new UnauthenticatedException("test", null, FakeStatusCode.of(Code.UNAUTHENTICATED), false); + call.getController().getObserver().onError(initialError); + + // Should notify the outer future + ExecutionException ee = + assertThrows( + ExecutionException.class, + () -> fakeRetryingFuture.getAttemptResult().get(1, TimeUnit.SECONDS)); + Throwable outerError = ee.getCause(); + Mockito.verify(transportChannel).refresh(); + Truth.assertThat(outerError).isInstanceOf(ServerStreamingAttemptException.class); + Truth.assertThat(((ServerStreamingAttemptException) outerError).hasSeenResponses()).isFalse(); + Truth.assertThat(((ServerStreamingAttemptException) outerError).canResume()).isTrue(); + Truth.assertThat(outerError.getCause()).isInstanceOf(UnauthenticatedException.class); + Truth.assertThat(((UnauthenticatedException) outerError.getCause()).isRetryable()).isFalse(); + Truth.assertThat(((UnauthenticatedException) outerError.getCause()).isChannelRefreshed()) + .isFalse(); + } + + @Test + @SuppressWarnings("ConstantConditions") + void testUnauthenticatedRefreshWithGenerationAdvance_flagsChannelRefreshed() { + TransportChannel transportChannel = Mockito.mock(TransportChannel.class); + AtomicLong generation = new AtomicLong(0); + Mockito.when(transportChannel.getGeneration()).thenAnswer(inv -> generation.get()); + Mockito.when(transportChannel.shouldRefresh()).thenReturn(true); + Mockito.doAnswer( + inv -> { + generation.incrementAndGet(); + return null; + }) + .when(transportChannel) + .refresh(); + + ApiCallContext context = Mockito.mock(ApiCallContext.class); + Mockito.when(context.getTransportChannel()).thenReturn(transportChannel); + Mockito.when(context.getTracer()).thenReturn(BaseApiTracer.getInstance()); + Mockito.when(context.getTimeoutDuration()).thenReturn(Duration.ofHours(5)); + + resumptionStrategy = new MyStreamResumptionStrategy(); + ServerStreamingAttemptCallable callable = createCallable(context); + callable.start(); + + MockServerStreamingCall call = innerCallable.popLastCall(); + + UnauthenticatedException initialError = + new UnauthenticatedException("test", null, FakeStatusCode.of(Code.UNAUTHENTICATED), false); + call.getController().getObserver().onError(initialError); + + ExecutionException ee = + assertThrows( + ExecutionException.class, + () -> fakeRetryingFuture.getAttemptResult().get(1, TimeUnit.SECONDS)); + Throwable outerError = ee.getCause(); + Mockito.verify(transportChannel).refresh(); + Truth.assertThat(outerError).isInstanceOf(ServerStreamingAttemptException.class); + Truth.assertThat(((ServerStreamingAttemptException) outerError).canResume()).isTrue(); + Truth.assertThat(outerError.getCause()).isInstanceOf(UnauthenticatedException.class); + Truth.assertThat(((UnauthenticatedException) outerError.getCause()).isChannelRefreshed()) + .isTrue(); + Truth.assertThat(((UnauthenticatedException) outerError.getCause()).isRetryable()).isFalse(); + Truth.assertThat(outerError.getCause().getStackTrace()).isEqualTo(initialError.getStackTrace()); + // A fixed clock keeps the fake attempt (started at t=0) within its total timeout. + StreamingRetryAlgorithm retryAlgorithm = + new StreamingRetryAlgorithm<>( + new ApiResultRetryAlgorithm<>(), + new ExponentialRetryAlgorithm(RetrySettings.newBuilder().build(), new FakeApiClock(0))); + // Mirror the retry executor: compute the next attempt, then ask whether to run it. + TimedAttemptSettings nextAttempt = + retryAlgorithm.createNextAttempt(outerError, null, fakeRetryingFuture.getAttemptSettings()); + Truth.assertThat(nextAttempt).isNotNull(); + Truth.assertThat(retryAlgorithm.shouldRetry(outerError, null, nextAttempt)).isTrue(); + + // Verify retry call resumes stream + callable.call(); + call = innerCallable.popLastCall(); + Truth.assertThat(call.getRequest()).isEqualTo("request > 0"); + } + + @Test + @SuppressWarnings("ConstantConditions") + void testUnauthenticatedRefreshWithNonResumableStreamDoesNotRetry() { + TransportChannel transportChannel = Mockito.mock(TransportChannel.class); + AtomicLong generation = new AtomicLong(0); + Mockito.when(transportChannel.getGeneration()).thenAnswer(inv -> generation.get()); + Mockito.when(transportChannel.shouldRefresh()).thenReturn(true); + Mockito.doAnswer( + inv -> { + generation.incrementAndGet(); + return null; + }) + .when(transportChannel) + .refresh(); + + ApiCallContext context = Mockito.mock(ApiCallContext.class); + Mockito.when(context.getTransportChannel()).thenReturn(transportChannel); + Mockito.when(context.getTracer()).thenReturn(BaseApiTracer.getInstance()); + Mockito.when(context.getTimeoutDuration()).thenReturn(Duration.ofHours(5)); + + // SimpleStreamResumptionStrategy cannot resume once a response has been received + resumptionStrategy = new SimpleStreamResumptionStrategy<>(); + ServerStreamingAttemptCallable callable = createCallable(context); + callable.start(); + + MockServerStreamingCall call = innerCallable.popLastCall(); + call.getController().getObserver().onResponse("response1"); + + UnauthenticatedException initialError = + new UnauthenticatedException("test", null, FakeStatusCode.of(Code.UNAUTHENTICATED), false); + call.getController().getObserver().onError(initialError); + + ExecutionException ee = + assertThrows( + ExecutionException.class, + () -> fakeRetryingFuture.getAttemptResult().get(1, TimeUnit.SECONDS)); + Throwable outerError = ee.getCause(); + Mockito.verify(transportChannel).refresh(); + Truth.assertThat(outerError).isInstanceOf(ServerStreamingAttemptException.class); + ServerStreamingAttemptException attemptEx = (ServerStreamingAttemptException) outerError; + Truth.assertThat(attemptEx.canResume()).isFalse(); + Truth.assertThat(((UnauthenticatedException) attemptEx.getCause()).isChannelRefreshed()) + .isTrue(); + Truth.assertThat( + new StreamingRetryAlgorithm<>( + new ApiResultRetryAlgorithm<>(), + new ExponentialRetryAlgorithm( + RetrySettings.newBuilder().build(), NanoClock.getDefaultClock())) + .shouldRetry(attemptEx, null, fakeRetryingFuture.getAttemptSettings())) + .isFalse(); + } + + @Test + @SuppressWarnings("ConstantConditions") + void testRefreshThrowsException_originalErrorNotLost() { + TransportChannel transportChannel = Mockito.mock(TransportChannel.class); + Mockito.when(transportChannel.shouldRefresh()).thenReturn(true); + Mockito.doThrow(new RuntimeException("Refresh error")).when(transportChannel).refresh(); + + ApiCallContext context = Mockito.mock(ApiCallContext.class); + Mockito.when(context.getTransportChannel()).thenReturn(transportChannel); + Mockito.when(context.getTracer()).thenReturn(BaseApiTracer.getInstance()); + Mockito.when(context.getTimeoutDuration()).thenReturn(Duration.ofHours(5)); + + resumptionStrategy = new MyStreamResumptionStrategy(); + ServerStreamingAttemptCallable callable = createCallable(context); + callable.start(); + + MockServerStreamingCall call = innerCallable.popLastCall(); + + UnauthenticatedException initialError = + new UnauthenticatedException("test", null, FakeStatusCode.of(Code.UNAUTHENTICATED), false); + call.getController().getObserver().onError(initialError); + + ExecutionException ee = + assertThrows( + ExecutionException.class, + () -> fakeRetryingFuture.getAttemptResult().get(1, TimeUnit.SECONDS)); + Throwable outerError = ee.getCause(); + Mockito.verify(transportChannel).refresh(); + Truth.assertThat(outerError).isInstanceOf(ServerStreamingAttemptException.class); + Truth.assertThat(outerError.getCause()).isEqualTo(initialError); + } + @Test @SuppressWarnings("ConstantConditions") void testMidRetry() { @@ -403,6 +597,139 @@ public String processResponse(String response) { .containsExactly("first+suffix", "second+suffix", "third+suffix"); } + @Test + void testUnauthenticatedException_whenChannelRefreshes_flagsChannelRefreshed() throws Exception { + FakeChannel fakeChannel = new FakeChannel(); + fakeChannel.setShouldRefresh(true); + ApiCallContext context = + FakeCallContext.createDefault() + .withTransportChannel(FakeTransportChannel.create(fakeChannel)); + + ServerStreamingAttemptCallable callable = createCallable(context); + callable.start(); + + MockServerStreamingCall call = innerCallable.popLastCall(); + UnauthenticatedException unauthEx = + new UnauthenticatedException( + "cert expired", null, new FakeStatusCode(Code.UNAUTHENTICATED), false); + call.getController().getObserver().onError(unauthEx); + + ExecutionException ex = + assertThrows( + ExecutionException.class, + () -> fakeRetryingFuture.getAttemptResult().get(1, TimeUnit.SECONDS)); + Truth.assertThat(ex.getCause()).isInstanceOf(ServerStreamingAttemptException.class); + Throwable cause = ex.getCause().getCause(); + Truth.assertThat(cause).isInstanceOf(UnauthenticatedException.class); + Truth.assertThat(((UnauthenticatedException) cause).isChannelRefreshed()).isTrue(); + Truth.assertThat(fakeChannel.getRefreshCount()).isEqualTo(1); + } + + @Test + void testUnauthenticatedException_whenChannelRefreshFails_notFlagged() throws Exception { + AtomicInteger refreshCalls = new AtomicInteger(); + FakeChannel fakeChannel = + new FakeChannel() { + @Override + public void refresh() { + refreshCalls.incrementAndGet(); + throw new RuntimeException("Refresh failed"); + } + }; + fakeChannel.setShouldRefresh(true); + ApiCallContext context = + FakeCallContext.createDefault() + .withTransportChannel(FakeTransportChannel.create(fakeChannel)); + + ServerStreamingAttemptCallable callable = createCallable(context); + callable.start(); + + MockServerStreamingCall call = innerCallable.popLastCall(); + UnauthenticatedException unauthEx = + new UnauthenticatedException( + "cert expired", null, new FakeStatusCode(Code.UNAUTHENTICATED), false); + call.getController().getObserver().onError(unauthEx); + + ExecutionException ex = + assertThrows( + ExecutionException.class, + () -> fakeRetryingFuture.getAttemptResult().get(1, TimeUnit.SECONDS)); + Truth.assertThat(refreshCalls.get()).isEqualTo(1); + Truth.assertThat(ex.getCause()).isInstanceOf(ServerStreamingAttemptException.class); + Throwable cause = ex.getCause().getCause(); + Truth.assertThat(cause).isInstanceOf(UnauthenticatedException.class); + Truth.assertThat(((UnauthenticatedException) cause).isRetryable()).isFalse(); + Truth.assertThat(((UnauthenticatedException) cause).isChannelRefreshed()).isFalse(); + } + + @Test + void testUnauthenticated_generationAlreadyAdvanced_flaggedWithoutCheckingShouldRefresh() + throws Exception { + TransportChannel transportChannel = Mockito.mock(TransportChannel.class); + AtomicLong generation = new AtomicLong(0); + Mockito.when(transportChannel.getGeneration()).thenAnswer(inv -> generation.get()); + + ApiCallContext context = Mockito.mock(ApiCallContext.class); + Mockito.when(context.getTransportChannel()).thenReturn(transportChannel); + Mockito.when(context.getTracer()).thenReturn(BaseApiTracer.getInstance()); + Mockito.when(context.getTimeoutDuration()).thenReturn(Duration.ofHours(5)); + + resumptionStrategy = new MyStreamResumptionStrategy(); + ServerStreamingAttemptCallable callable = createCallable(context); + callable.start(); + + MockServerStreamingCall call = innerCallable.popLastCall(); + // Another request rotated the channel while this stream was open. + generation.incrementAndGet(); + call.getController() + .getObserver() + .onError( + new UnauthenticatedException( + "cert expired", null, new FakeStatusCode(Code.UNAUTHENTICATED), false)); + + ExecutionException ex = + assertThrows( + ExecutionException.class, + () -> fakeRetryingFuture.getAttemptResult().get(1, TimeUnit.SECONDS)); + Throwable cause = ex.getCause().getCause(); + Truth.assertThat(cause).isInstanceOf(UnauthenticatedException.class); + Truth.assertThat(((UnauthenticatedException) cause).isChannelRefreshed()).isTrue(); + Mockito.verify(transportChannel, Mockito.never()).shouldRefresh(); + Mockito.verify(transportChannel, Mockito.never()).refresh(); + } + + @Test + void testUnauthenticated_shouldRefreshThrows_originalErrorPreserved() throws Exception { + FakeChannel fakeChannel = + new FakeChannel() { + @Override + public boolean shouldRefresh() { + throw new IllegalStateException("Unable to read certificate"); + } + }; + ApiCallContext context = + FakeCallContext.createDefault() + .withTransportChannel(FakeTransportChannel.create(fakeChannel)); + + ServerStreamingAttemptCallable callable = createCallable(context); + callable.start(); + + MockServerStreamingCall call = innerCallable.popLastCall(); + UnauthenticatedException unauthEx = + new UnauthenticatedException( + "cert expired", null, new FakeStatusCode(Code.UNAUTHENTICATED), false); + call.getController().getObserver().onError(unauthEx); + + ExecutionException ex = + assertThrows( + ExecutionException.class, + () -> fakeRetryingFuture.getAttemptResult().get(1, TimeUnit.SECONDS)); + Truth.assertThat(ex.getCause()).isInstanceOf(ServerStreamingAttemptException.class); + Truth.assertThat(ex.getCause().getCause()).isSameInstanceAs(unauthEx); + Truth.assertThat(unauthEx.isChannelRefreshed()).isFalse(); + Truth.assertThat(fakeChannel.getRefreshCount()).isEqualTo(0); + } + static class MyStreamResumptionStrategy implements StreamResumptionStrategy { private int responseCount; diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/StreamingCallableTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/StreamingCallableTest.java index 6f9584826893..92d2b3a9cc8e 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/StreamingCallableTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/StreamingCallableTest.java @@ -29,6 +29,7 @@ */ package com.google.api.gax.rpc; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.mockito.Mockito.mock; @@ -129,7 +130,7 @@ void testClientStreamingCall() { ClientStreamingCallable callable = stashCallable.withDefaultCallContext(defaultCallContext); callable.clientStreamingCall(observer); - assertSame(observer, stashCallable.getActualObserver()); + assertNotNull(stashCallable.getActualObserver()); assertSame(defaultCallContext, stashCallable.getContext()); } @@ -158,7 +159,7 @@ void testClientStreamingCallWithContext() { ClientStreamingCallable callable = stashCallable.withDefaultCallContext(FakeCallContext.createDefault()); callable.clientStreamingCall(observer, context); - assertSame(observer, stashCallable.getActualObserver()); + assertNotNull(stashCallable.getActualObserver()); FakeCallContext actualContext = (FakeCallContext) stashCallable.getContext(); assertSame(channel, actualContext.getChannel()); assertSame(credentials, actualContext.getCredentials()); diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/UnauthenticatedExceptionTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/UnauthenticatedExceptionTest.java new file mode 100644 index 000000000000..083de5286277 --- /dev/null +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/UnauthenticatedExceptionTest.java @@ -0,0 +1,183 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.rpc; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.rpc.testing.FakeStatusCode; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.ObjectStreamClass; +import java.io.Serializable; +import java.util.Base64; +import java.util.Collections; +import org.junit.jupiter.api.Test; + +class UnauthenticatedExceptionTest { + + /** + * An {@link UnauthenticatedException} with message "serialized by released gax", no cause, an + * empty stack trace, retryable {@code true} and a {@link SerializableStatusCode} of + * UNAUTHENTICATED, Java-serialized with the released gax 2.87.0 jar. + */ + private static final String SERIALIZED_BY_GAX_2_87_0 = + "rO0ABXNyAC9jb20uZ29vZ2xlLmFwaS5nYXgucnBjLlVuYXV0aGVudGljYXRlZEV4Y2VwdGlvbmC+YDhKxcJlAgAAeHIAI2NvbS5nb29nbGUuYXBpLmdheC5ycGMuQXBpRXhjZXB0aW9uw0h4sCzSVFQCAANaAAlyZXRyeWFibGVMAAxlcnJvckRldGFpbHN0ACVMY29tL2dvb2dsZS9hcGkvZ2F4L3JwYy9FcnJvckRldGFpbHM7TAAKc3RhdHVzQ29kZXQAI0xjb20vZ29vZ2xlL2FwaS9nYXgvcnBjL1N0YXR1c0NvZGU7eHIAGmphdmEubGFuZy5SdW50aW1lRXhjZXB0aW9unl8GRwo0g+UCAAB4cgATamF2YS5sYW5nLkV4Y2VwdGlvbtD9Hz4aOxzEAgAAeHIAE2phdmEubGFuZy5UaHJvd2FibGXVxjUnOXe4ywMABEwABWNhdXNldAAVTGphdmEvbGFuZy9UaHJvd2FibGU7TAANZGV0YWlsTWVzc2FnZXQAEkxqYXZhL2xhbmcvU3RyaW5nO1sACnN0YWNrVHJhY2V0AB5bTGphdmEvbGFuZy9TdGFja1RyYWNlRWxlbWVudDtMABRzdXBwcmVzc2VkRXhjZXB0aW9uc3QAEExqYXZhL3V0aWwvTGlzdDt4cHB0ABpzZXJpYWxpemVkIGJ5IHJlbGVhc2VkIGdheHVyAB5bTGphdmEubGFuZy5TdGFja1RyYWNlRWxlbWVudDsCRio8PP0iOQIAAHhwAAAAAHNyAB9qYXZhLnV0aWwuQ29sbGVjdGlvbnMkRW1wdHlMaXN0ergXtDynnt4CAAB4cHgBcHNyAEpjb20uZ29vZ2xlLmFwaS5nYXgucnBjLlVuYXV0aGVudGljYXRlZEV4Y2VwdGlvblRlc3QkU2VyaWFsaXphYmxlU3RhdHVzQ29kZQAAAAAAAAABAgABTAAEY29kZXQAKExjb20vZ29vZ2xlL2FwaS9nYXgvcnBjL1N0YXR1c0NvZGUkQ29kZTt4cH5yACZjb20uZ29vZ2xlLmFwaS5nYXgucnBjLlN0YXR1c0NvZGUkQ29kZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAD1VOQVVUSEVOVElDQVRFRA=="; + + /** A serializable {@link StatusCode}, since the transport implementations are not. */ + static final class SerializableStatusCode implements StatusCode, Serializable { + private static final long serialVersionUID = 1L; + private final Code code; + + SerializableStatusCode(Code code) { + this.code = code; + } + + @Override + public Code getCode() { + return code; + } + + @Override + public Object getTransportCode() { + return code.name(); + } + } + + @Test + void publicConstructors_areNotChannelRefreshed() { + StatusCode statusCode = FakeStatusCode.of(StatusCode.Code.UNAUTHENTICATED); + ErrorDetails errorDetails = + ErrorDetails.builder().setRawErrorMessages(Collections.emptyList()).build(); + + assertThat(new UnauthenticatedException(null, statusCode, true).isChannelRefreshed()).isFalse(); + assertThat(new UnauthenticatedException("msg", null, statusCode, true).isChannelRefreshed()) + .isFalse(); + assertThat( + new UnauthenticatedException(null, statusCode, true, errorDetails).isChannelRefreshed()) + .isFalse(); + assertThat( + new UnauthenticatedException("msg", null, statusCode, true, errorDetails) + .isChannelRefreshed()) + .isFalse(); + } + + @Test + void withChannelRefreshed_preservesFields() { + ErrorDetails errorDetails = + ErrorDetails.builder().setRawErrorMessages(Collections.emptyList()).build(); + IllegalStateException cause = new IllegalStateException("root cause"); + StatusCode statusCode = FakeStatusCode.of(StatusCode.Code.UNAUTHENTICATED); + UnauthenticatedException original = + new UnauthenticatedException("Expired cert", cause, statusCode, false, errorDetails); + original.setStackTrace( + new StackTraceElement[] {new StackTraceElement("foo", "bar", "Baz.java", 123)}); + RuntimeException suppressed = new RuntimeException("suppressed"); + original.addSuppressed(suppressed); + + UnauthenticatedException refreshed = original.withChannelRefreshed(); + + assertThat(refreshed).isNotSameInstanceAs(original); + assertThat(refreshed.isChannelRefreshed()).isTrue(); + assertThat(original.isChannelRefreshed()).isFalse(); + assertThat(refreshed.getMessage()).isEqualTo(original.getMessage()); + assertThat(refreshed.getCause()).isSameInstanceAs(cause); + assertThat(refreshed.getStatusCode()).isSameInstanceAs(statusCode); + assertThat(refreshed.getErrorDetails()).isSameInstanceAs(errorDetails); + assertThat(refreshed.getStackTrace()).isEqualTo(original.getStackTrace()); + assertThat(refreshed.getSuppressed()).asList().containsExactly(suppressed); + } + + @Test + void withChannelRefreshed_keepsIsRetryable() { + StatusCode statusCode = FakeStatusCode.of(StatusCode.Code.UNAUTHENTICATED); + + assertThat( + new UnauthenticatedException("msg", null, statusCode, false) + .withChannelRefreshed() + .isRetryable()) + .isFalse(); + assertThat( + new UnauthenticatedException("msg", null, statusCode, true) + .withChannelRefreshed() + .isRetryable()) + .isTrue(); + } + + @Test + void serialVersionUID_matchesReleasedValue() { + // gax 2.83.0 to 2.87.0 did not declare a serialVersionUID; this is the value the JVM computed + // for them. Keeping it avoids InvalidClassException across versions. + assertThat(ObjectStreamClass.lookup(UnauthenticatedException.class).getSerialVersionUID()) + .isEqualTo(6971115068105015909L); + } + + @Test + void deserializesExceptionSerializedByReleasedGax() throws Exception { + Object deserialized = deserialize(Base64.getDecoder().decode(SERIALIZED_BY_GAX_2_87_0)); + + assertThat(deserialized).isInstanceOf(UnauthenticatedException.class); + UnauthenticatedException ex = (UnauthenticatedException) deserialized; + assertThat(ex.getMessage()).isEqualTo("serialized by released gax"); + assertThat(ex.getStatusCode().getCode()).isEqualTo(StatusCode.Code.UNAUTHENTICATED); + assertThat(ex.isRetryable()).isTrue(); + assertThat(ex.isChannelRefreshed()).isFalse(); + } + + @Test + void withChannelRefreshed_flagNotSerialized() throws Exception { + UnauthenticatedException refreshed = + new UnauthenticatedException( + "msg", null, new SerializableStatusCode(StatusCode.Code.UNAUTHENTICATED), false) + .withChannelRefreshed(); + + UnauthenticatedException roundTripped = + (UnauthenticatedException) deserialize(serialize(refreshed)); + + assertThat(roundTripped.getMessage()).isEqualTo("msg"); + assertThat(roundTripped.isRetryable()).isFalse(); + assertThat(roundTripped.isChannelRefreshed()).isFalse(); + } + + private static byte[] serialize(Object object) throws Exception { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream out = new ObjectOutputStream(bytes)) { + out.writeObject(object); + } + return bytes.toByteArray(); + } + + private static Object deserialize(byte[] bytes) throws Exception { + try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(bytes))) { + return in.readObject(); + } + } +} diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/mtls/AbstractMtlsTransportChannelTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/mtls/AbstractMtlsTransportChannelTest.java index bea4674b765b..3bdfcc7c1ab6 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/mtls/AbstractMtlsTransportChannelTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/mtls/AbstractMtlsTransportChannelTest.java @@ -40,6 +40,7 @@ import java.io.IOException; import java.security.GeneralSecurityException; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; public abstract class AbstractMtlsTransportChannelTest { /** @@ -63,9 +64,10 @@ void testNotUseClientCertificate() throws IOException, GeneralSecurityException @Test void testUseClientCertificate() throws IOException, GeneralSecurityException { - CertificateBasedAccess certificateBasedAccess = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "auto" : "true"); + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.getMtlsEndpointUsagePolicy()) + .thenReturn(CertificateBasedAccess.MtlsEndpointUsagePolicy.AUTO); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); MtlsProvider provider = new FakeMtlsProvider(FakeMtlsProvider.createTestMtlsKeyStore(), "", false); assertNotNull(getMtlsObjectFromTransportChannel(provider, certificateBasedAccess)); @@ -73,9 +75,10 @@ void testUseClientCertificate() throws IOException, GeneralSecurityException { @Test void testNoClientCertificate() throws IOException, GeneralSecurityException { - CertificateBasedAccess certificateBasedAccess = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "auto" : "true"); + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.getMtlsEndpointUsagePolicy()) + .thenReturn(CertificateBasedAccess.MtlsEndpointUsagePolicy.AUTO); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); MtlsProvider provider = new FakeMtlsProvider(null, "", false); assertNull(getMtlsObjectFromTransportChannel(provider, certificateBasedAccess)); } @@ -83,9 +86,10 @@ void testNoClientCertificate() throws IOException, GeneralSecurityException { @Test void testGetKeyStoreThrows() throws GeneralSecurityException { // Test the case where provider.getKeyStore() throws. - CertificateBasedAccess certificateBasedAccess = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "auto" : "true"); + CertificateBasedAccess certificateBasedAccess = Mockito.mock(CertificateBasedAccess.class); + Mockito.when(certificateBasedAccess.getMtlsEndpointUsagePolicy()) + .thenReturn(CertificateBasedAccess.MtlsEndpointUsagePolicy.AUTO); + Mockito.when(certificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); MtlsProvider provider = new FakeMtlsProvider(null, "", true); IOException actual = assertThrows( diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/mtls/CertificateBasedAccessTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/mtls/CertificateBasedAccessTest.java index e328e0af4799..44c23ce2c9b5 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/mtls/CertificateBasedAccessTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/mtls/CertificateBasedAccessTest.java @@ -32,52 +32,133 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; import org.junit.jupiter.api.Test; class CertificateBasedAccessTest { + private static class TestEnv { + private final Map env = new HashMap<>(); + + TestEnv() { + // Hermetically isolate tests from the host's ~/.config/gcloud/certificate_config.json + env.put("CLOUDSDK_CONFIG", "/nonexistent/test/gcloud"); + } + + void set(String key, String val) { + env.put(key, val); + } + + String get(String name) { + return env.get(name); + } + } + + private CertificateBasedAccess createCba(TestEnv env) { + return new CertificateBasedAccess(env::get); + } + @Test void testUseMtlsEndpointAlways() { - CertificateBasedAccess cba = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "always" : "false"); + TestEnv env = new TestEnv(); + env.set("GOOGLE_API_USE_MTLS_ENDPOINT", "always"); + CertificateBasedAccess cba = createCba(env); assertEquals( CertificateBasedAccess.MtlsEndpointUsagePolicy.ALWAYS, cba.getMtlsEndpointUsagePolicy()); } @Test void testUseMtlsEndpointAuto() { - CertificateBasedAccess cba = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "auto" : "false"); + TestEnv env = new TestEnv(); + env.set("GOOGLE_API_USE_MTLS_ENDPOINT", "auto"); + CertificateBasedAccess cba = createCba(env); assertEquals( CertificateBasedAccess.MtlsEndpointUsagePolicy.AUTO, cba.getMtlsEndpointUsagePolicy()); } @Test void testUseMtlsEndpointNever() { - CertificateBasedAccess cba = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_MTLS_ENDPOINT") ? "never" : "false"); + TestEnv env = new TestEnv(); + env.set("GOOGLE_API_USE_MTLS_ENDPOINT", "never"); + CertificateBasedAccess cba = createCba(env); + assertEquals( + CertificateBasedAccess.MtlsEndpointUsagePolicy.NEVER, cba.getMtlsEndpointUsagePolicy()); + } + + @Test + void testUseMtlsEndpointCaseInsensitive() { + TestEnv env = new TestEnv(); + env.set("GOOGLE_API_USE_MTLS_ENDPOINT", "ALWAYS"); + CertificateBasedAccess cba = createCba(env); + assertEquals( + CertificateBasedAccess.MtlsEndpointUsagePolicy.ALWAYS, cba.getMtlsEndpointUsagePolicy()); + + env.set("GOOGLE_API_USE_MTLS_ENDPOINT", "NEVER"); assertEquals( CertificateBasedAccess.MtlsEndpointUsagePolicy.NEVER, cba.getMtlsEndpointUsagePolicy()); } @Test - void testUseMtlsClientCertificateTrue() { - CertificateBasedAccess cba = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_CLIENT_CERTIFICATE") ? "true" : "auto"); + void testUseMtlsClientCertificateExplicitTrueNoCredentials() { + TestEnv env = new TestEnv(); + env.set("GOOGLE_API_USE_CLIENT_CERTIFICATE", "true"); + CertificateBasedAccess cba = createCba(env); + // Explicit 'true' enables mTLS client certificate usage (for ECP / custom MtlsProvider) even + // when no workload cert files are present, while getWorkloadCertPath returns null so file + // rotation polling is not active. assertTrue(cba.useMtlsClientCertificate()); + assertNull(cba.getWorkloadCertPath()); } @Test - void testUseMtlsClientCertificateFalse() { - CertificateBasedAccess cba = - new CertificateBasedAccess( - name -> name.equals("GOOGLE_API_USE_CLIENT_CERTIFICATE") ? "false" : "auto"); + void testUseMtlsClientCertificateExplicitFalse() { + TestEnv env = new TestEnv(); + env.set("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false"); + + CertificateBasedAccess cba = createCba(env); + assertFalse(cba.useMtlsClientCertificate()); + assertNull(cba.getWorkloadCertPath()); + } + + @Test + void testUseMtlsClientCertificateUnsetNoFiles() { + TestEnv env = new TestEnv(); + CertificateBasedAccess cba = createCba(env); assertFalse(cba.useMtlsClientCertificate()); + assertNull(cba.getWorkloadCertPath()); + } + + @Test + void testUseMtlsClientCertificateConfigMissingConfigFile_throwsIllegalStateException() { + TestEnv env = new TestEnv(); + env.set("GOOGLE_API_CERTIFICATE_CONFIG", "/nonexistent/config.json"); + + CertificateBasedAccess cba = createCba(env); + + // Non-existent config file on disk specified via explicit env var throws IllegalStateException + // (Fail Closed) + assertThrows(IllegalStateException.class, () -> cba.useMtlsClientCertificate()); + assertThrows(IllegalStateException.class, () -> cba.getWorkloadCertPath()); + } + + @Test + void testWorkloadCertificateUtilsEmptyFileReturnsEmptyString() throws Exception { + File tempFile = File.createTempFile("test-cert-empty", ".pem"); + tempFile.deleteOnExit(); + // 0-byte truncated file mid-write should return empty string rather than SHA-256 of empty bytes + assertEquals( + "", WorkloadCertificateUtils.getCertificateFingerprint(tempFile.getAbsolutePath())); + + Files.write(tempFile.toPath(), "test-cert-content".getBytes(StandardCharsets.UTF_8)); + String fp = WorkloadCertificateUtils.getCertificateFingerprint(tempFile.getAbsolutePath()); + assertFalse(fp.isEmpty()); } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/mtls/CertificateRotationTrackerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/mtls/CertificateRotationTrackerTest.java new file mode 100644 index 000000000000..f1fe388039b9 --- /dev/null +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/mtls/CertificateRotationTrackerTest.java @@ -0,0 +1,345 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.rpc.mtls; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import org.junit.jupiter.api.Test; + +class CertificateRotationTrackerTest { + + private static final String CERT_PATH = "/fake/cert/path"; + + /** Fingerprint reader backed by {@link #fingerprint} that counts disk reads. */ + private final AtomicReference fingerprint = new AtomicReference<>("F1"); + + private final AtomicInteger reads = new AtomicInteger(); + private final Function reader = + path -> { + assertEquals(CERT_PATH, path); + reads.incrementAndGet(); + return fingerprint.get(); + }; + + private CertificateRotationTracker newTracker() { + return new CertificateRotationTracker(() -> CERT_PATH, reader); + } + + @Test + void constructor_readsBaselineFingerprint() { + CertificateRotationTracker tracker = newTracker(); + + assertEquals("F1", tracker.getActiveCertFingerprint()); + assertEquals(1, reads.get()); + } + + @Test + void constructor_withoutCertPath_hasEmptyFingerprintAndNeverReadsDisk() { + CertificateRotationTracker tracker = new CertificateRotationTracker(() -> null, reader); + + assertEquals("", tracker.getActiveCertFingerprint()); + assertFalse(tracker.shouldRefresh()); + assertEquals("", tracker.readDiskFingerprint()); + assertEquals(0, reads.get()); + } + + @Test + void constructor_withUnreadableCert_hasEmptyFingerprint() { + fingerprint.set(null); + + assertEquals("", newTracker().getActiveCertFingerprint()); + } + + @Test + void shouldRefresh_whenCertUnchanged_returnsFalse() { + assertFalse(newTracker().shouldRefresh()); + } + + @Test + void shouldRefresh_whenCertChanged_returnsTrue() { + CertificateRotationTracker tracker = newTracker(); + fingerprint.set("F2"); + + assertTrue(tracker.shouldRefresh()); + } + + @Test + void shouldRefresh_comparesFingerprintsCaseInsensitively() { + fingerprint.set("abcdef"); + CertificateRotationTracker tracker = newTracker(); + fingerprint.set("ABCDEF"); + + assertFalse(tracker.shouldRefresh()); + } + + @Test + void shouldRefresh_whenCertUnreadable_returnsFalse() { + CertificateRotationTracker tracker = newTracker(); + fingerprint.set(""); + + assertFalse(tracker.shouldRefresh()); + } + + @Test + void shouldRefresh_cachesDetectedRotation() { + CertificateRotationTracker tracker = newTracker(); + fingerprint.set("F2"); + + assertTrue(tracker.shouldRefresh()); + assertTrue(tracker.shouldRefresh()); + // One baseline read plus one disk check; the second call is served from the cache. + assertEquals(2, reads.get()); + } + + @Test + void shouldRefresh_doesNotCacheUnchangedResult() { + CertificateRotationTracker tracker = newTracker(); + + assertFalse(tracker.shouldRefresh()); + fingerprint.set("F2"); + + // The rotation is detected immediately rather than after a cache TTL. + assertTrue(tracker.shouldRefresh()); + assertEquals(3, reads.get()); + } + + @Test + void readDiskFingerprint_bypassesCache() { + CertificateRotationTracker tracker = newTracker(); + fingerprint.set("F2"); + assertTrue(tracker.shouldRefresh()); + fingerprint.set("F3"); + + assertEquals("F3", tracker.readDiskFingerprint()); + } + + @Test + void readDiskFingerprint_whenCertUnreadable_returnsEmpty() { + CertificateRotationTracker tracker = newTracker(); + fingerprint.set(null); + + assertEquals("", tracker.readDiskFingerprint()); + } + + @Test + void markRefreshed_updatesActiveFingerprintAndClearsCachedRotation() { + CertificateRotationTracker tracker = newTracker(); + fingerprint.set("F2"); + assertTrue(tracker.shouldRefresh()); + + tracker.markRefreshed("F2"); + + assertEquals("F2", tracker.getActiveCertFingerprint()); + assertTrue(tracker.isAlreadyActive("F2")); + assertFalse(tracker.shouldRefresh()); + } + + @Test + void markRefreshed_ignoresNullAndEmptyFingerprints() { + CertificateRotationTracker tracker = newTracker(); + + tracker.markRefreshed(null); + tracker.markRefreshed(""); + + assertEquals("F1", tracker.getActiveCertFingerprint()); + } + + @Test + void isAlreadyActive_matchesActiveFingerprintCaseInsensitively() { + fingerprint.set("abcdef"); + CertificateRotationTracker tracker = newTracker(); + + assertTrue(tracker.isAlreadyActive("abcdef")); + assertTrue(tracker.isAlreadyActive("ABCDEF")); + assertFalse(tracker.isAlreadyActive("other")); + assertFalse(tracker.isAlreadyActive(null)); + } + + @Test + void invalidateCache_forcesNextCheckToReadDisk() { + CertificateRotationTracker tracker = newTracker(); + fingerprint.set("F2"); + assertTrue(tracker.shouldRefresh()); + // The certificate is rolled back while the rotation is cached. + fingerprint.set("F1"); + assertTrue(tracker.shouldRefresh()); + + tracker.invalidateCache(); + + assertFalse(tracker.shouldRefresh()); + } + + @Test + void shouldRefresh_rereadsDiskOnceCachedRotationExpires() throws InterruptedException { + CertificateRotationTracker tracker = newTracker(); + fingerprint.set("F2"); + assertTrue(tracker.shouldRefresh()); + // The certificate becomes unreadable while the rotation is cached; the cached rotation is + // still reported until it expires. + fingerprint.set(""); + assertTrue(tracker.shouldRefresh()); + + // Wait past the 1 second cache TTL. + Thread.sleep(1100); + + // The expired entry is not used: the disk is read again, and an unreadable certificate is not + // reported as a rotation. + assertFalse(tracker.shouldRefresh()); + fingerprint.set("F1"); + assertFalse(tracker.shouldRefresh()); + } + + @Test + void shouldRefresh_whenCertBecomesReadableAfterUnreadableBaseline_detectsRotation() { + // For example, the certificate had not been written yet when the channel was created. + fingerprint.set(""); + CertificateRotationTracker tracker = newTracker(); + assertEquals("", tracker.getActiveCertFingerprint()); + assertFalse(tracker.shouldRefresh()); + + fingerprint.set("F1"); + + assertTrue(tracker.shouldRefresh()); + tracker.markRefreshed("F1"); + assertFalse(tracker.shouldRefresh()); + } + + @Test + void stringConstructor_withNullOrMissingCert_hasEmptyFingerprintAndNoRotation() { + for (String certPath : new String[] {null, "/nonexistent/workload-cert.pem"}) { + CertificateRotationTracker tracker = new CertificateRotationTracker(certPath); + + assertEquals("", tracker.getActiveCertFingerprint()); + assertEquals("", tracker.readDiskFingerprint()); + assertFalse(tracker.shouldRefresh()); + } + } + + /** + * Creates a tracker whose first disk check after the constructor's baseline read blocks, after + * reading the current fingerprint, until {@code releaseRead} is counted down. + */ + private CertificateRotationTracker newTrackerWithBlockingDiskCheck( + CountDownLatch readStarted, CountDownLatch releaseRead) { + return new CertificateRotationTracker( + () -> CERT_PATH, + path -> { + String read = reader.apply(path); + if (reads.get() == 2) { + readStarted.countDown(); + try { + releaseRead.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + return read; + }); + } + + /** Waits until {@code thread} is parked waiting for a lock, or has finished. */ + private static void awaitWaitingOrTerminated(Thread thread) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (thread.getState() != Thread.State.WAITING + && thread.getState() != Thread.State.TERMINATED) { + assertTrue(System.nanoTime() < deadline); + Thread.sleep(1); + } + } + + @Test + void shouldRefresh_concurrentCallersShareOneDiskReadOfRotation() throws Exception { + CountDownLatch readStarted = new CountDownLatch(1); + CountDownLatch releaseRead = new CountDownLatch(1); + CertificateRotationTracker tracker = newTrackerWithBlockingDiskCheck(readStarted, releaseRead); + fingerprint.set("F2"); + AtomicReference firstResult = new AtomicReference<>(); + AtomicReference secondResult = new AtomicReference<>(); + + Thread first = new Thread(() -> firstResult.set(tracker.shouldRefresh())); + Thread second = new Thread(() -> secondResult.set(tracker.shouldRefresh())); + try { + first.start(); + assertTrue(readStarted.await(5, TimeUnit.SECONDS)); + // Another failing RPC checks for a rotation while the first disk check is in progress. + second.start(); + awaitWaitingOrTerminated(second); + } finally { + releaseRead.countDown(); + } + first.join(5000); + second.join(5000); + assertFalse(first.isAlive()); + assertFalse(second.isAlive()); + + assertEquals(Boolean.TRUE, firstResult.get()); + assertEquals(Boolean.TRUE, secondResult.get()); + // One baseline read plus a single disk check shared by both callers. + assertEquals(2, reads.get()); + } + + @Test + void markRefreshed_duringInProgressDiskCheck_doesNotLeaveStaleCachedFingerprint() + throws Exception { + CountDownLatch readStarted = new CountDownLatch(1); + CountDownLatch releaseRead = new CountDownLatch(1); + CertificateRotationTracker tracker = newTrackerWithBlockingDiskCheck(readStarted, releaseRead); + + Thread diskCheckThread = new Thread(tracker::shouldRefresh); + Thread markThread = new Thread(() -> tracker.markRefreshed("F2")); + try { + diskCheckThread.start(); + assertTrue(readStarted.await(5, TimeUnit.SECONDS)); + // The certificate rotates and a refresh switches to it while the disk check that read the + // old certificate is still in progress. + fingerprint.set("F2"); + markThread.start(); + awaitWaitingOrTerminated(markThread); + } finally { + releaseRead.countDown(); + } + diskCheckThread.join(5000); + markThread.join(5000); + assertFalse(diskCheckThread.isAlive()); + assertFalse(markThread.isAlive()); + + // The old fingerprint read by the in-progress check must not be left in the cache, where it + // would make shouldRefresh() report a rotation that has already been handled. + assertEquals("F2", tracker.getActiveCertFingerprint()); + assertFalse(tracker.shouldRefresh()); + } +} diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/testing/FakeCallContext.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/testing/FakeCallContext.java index 63ffcbeeccd3..2ad3292f580a 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/testing/FakeCallContext.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/testing/FakeCallContext.java @@ -240,6 +240,11 @@ public FakeChannel getChannel() { return channel; } + @Override + public TransportChannel getTransportChannel() { + return channel != null ? FakeTransportChannel.create(channel) : null; + } + @Override public java.time.Duration getTimeoutDuration() { return timeout; diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/testing/FakeChannel.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/testing/FakeChannel.java index b725da363b3b..9acfc703b9c3 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/testing/FakeChannel.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/testing/FakeChannel.java @@ -32,4 +32,36 @@ import com.google.api.core.InternalApi; @InternalApi("for testing") -public class FakeChannel {} +public class FakeChannel { + private volatile boolean shouldRefresh = false; + private volatile int refreshCount = 0; + + public FakeChannel setShouldRefresh(boolean shouldRefresh) { + this.shouldRefresh = shouldRefresh; + return this; + } + + public boolean shouldRefresh() { + return shouldRefresh; + } + + public void refresh() { + refreshCount++; + generation++; + } + + public int getRefreshCount() { + return refreshCount; + } + + private volatile long generation = 0; + + public FakeChannel setGeneration(long generation) { + this.generation = generation; + return this; + } + + public long getGeneration() { + return generation; + } +} diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/testing/FakeTransportChannel.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/testing/FakeTransportChannel.java index 0d4abac8f1c6..7d92556b88d5 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/testing/FakeTransportChannel.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/testing/FakeTransportChannel.java @@ -41,6 +41,48 @@ public class FakeTransportChannel implements TransportChannel { private volatile boolean isShutdown = false; private volatile Map headers; private volatile Executor executor; + private volatile boolean shouldRefresh = false; + private volatile int refreshCount = 0; + + public FakeTransportChannel setShouldRefresh(boolean shouldRefresh) { + if (channel != null) { + channel.setShouldRefresh(shouldRefresh); + } + this.shouldRefresh = shouldRefresh; + return this; + } + + @Override + public boolean shouldRefresh() { + return channel != null ? channel.shouldRefresh() : shouldRefresh; + } + + @Override + public void refresh() { + if (channel != null) { + channel.refresh(); + } + refreshCount++; + } + + public int getRefreshCount() { + return channel != null ? channel.getRefreshCount() : refreshCount; + } + + private volatile long generation = 0; + + public FakeTransportChannel setGeneration(long generation) { + if (channel != null) { + channel.setGeneration(generation); + } + this.generation = generation; + return this; + } + + @Override + public long getGeneration() { + return channel != null ? channel.getGeneration() : generation; + } private FakeTransportChannel(FakeChannel channel) { this.channel = channel;