Merge branch 'main' into migration.python-bigquery-magics.migration.2026-03-02_16-59-45.migrate

Author: chalmerlowe

.gitignore

@@ -57,7 +57,8 @@ coverage.xml
 *sponge_log.xml
 
 # System test environment variables.
-system_tests/local_test_setup
+**/system_tests/local_test_setup
+**/system_tests/data
 
 # Make sure a generated file isn't accidentally committed.
 pylintrc

.kokoro/system-single.sh

@@ -22,7 +22,14 @@ pwd
 
 # If NOX_SESSION is set, it only runs the specified session,
 # otherwise run all the sessions.
-SESSION_ARG=""
+NOX_SESSION_ARG=""
 
-[[ -z "${NOX_SESSION}" ]] || SESSION_ARG="-s ${NOX_SESSION}"
-python3 -m nox ${SESSION_ARG}
+# IF NOX_FILE is set, it runs the specific nox file,
+# otherwise it runs noxfile.py in the package directory.
+NOX_FILE_ARG=""
+
+[[ -z "${NOX_SESSION}" ]] || NOX_SESSION_ARG="-s ${NOX_SESSION}"
+
+[[ -z "${NOX_FILE}" ]] || NOX_FILE_ARG="-f ${NOX_FILE}"
+
+python3 -m nox ${NOX_SESSION_ARG} $NOX_FILE_ARG

.kokoro/system.sh

@@ -24,26 +24,69 @@ export PYTHONUNBUFFERED=1
 # Setup firestore account credentials
 export FIRESTORE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/firebase-credentials.json
 
-# Setup service account credentials.
-export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json
-
-# Setup project id.
-export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
-
-RETVAL=0
-
 export PROJECT_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/..)
 
 cd "$PROJECT_ROOT"
 
-pwd
-
-# A file for running system tests
-system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh"
-
 # This is needed in order for `git diff` to succeed
 git config --global --add safe.directory $(realpath .)
 
+RETVAL=0
+
+pwd
+
+run_package_test() {
+  local package_name=$1
+  local package_path="packages/${package_name}"
+  
+  # Declare local overrides to prevent bleeding into the next loop iteration
+  local PROJECT_ID
+  local GOOGLE_APPLICATION_CREDENTIALS
+  local NOX_FILE
+  local NOX_SESSION
+
+  echo "------------------------------------------------------------"
+  echo "Configuring environment for: ${package_name}"
+  echo "------------------------------------------------------------"
+
+  case "${package_name}" in
+    "google-auth")
+      # Copy files needed for google-auth system tests
+      mkdir -p "${package_path}/system_tests/data"
+      cp "${KOKORO_GFILE_DIR}/google-auth-service-account.json" "${package_path}/system_tests/data/service_account.json"
+      cp "${KOKORO_GFILE_DIR}/google-auth-authorized-user.json" "${package_path}/system_tests/data/authorized_user.json"
+      cp "${KOKORO_GFILE_DIR}/google-auth-impersonated-service-account.json" "${package_path}/system_tests/data/impersonated_service_account.json"
+
+      PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/google-auth-project-id.json")
+      GOOGLE_APPLICATION_CREDENTIALS="${KOKORO_GFILE_DIR}/google-auth-service-account.json"
+      NOX_FILE="system_tests/noxfile.py"
+      NOX_SESSION=""
+      ;;
+    *)
+      PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
+      GOOGLE_APPLICATION_CREDENTIALS="${KOKORO_GFILE_DIR}/service-account.json"
+      NOX_FILE="noxfile.py"
+      NOX_SESSION="system-3.12"
+      ;;
+  esac
+
+  # Export variables for the duration of this function's sub-processes
+  export PROJECT_ID GOOGLE_APPLICATION_CREDENTIALS NOX_FILE NOX_SESSION
+
+  gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS"
+  gcloud config set project "$PROJECT_ID"
+
+  # Run the actual test
+  pushd "${package_path}" > /dev/null
+  set +e
+  "${system_test_script}"
+  local res=$?
+  set -e
+  popd > /dev/null
+  
+  return $res
+}
+
 packages_with_system_tests=(
   "google-auth"
   "google-cloud-bigquery-storage"
@@ -56,42 +99,38 @@ packages_with_system_tests=(
   "google-cloud-testutils"
 )
 
+# A file for running system tests
+system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh"
+
 # Join array elements with | for the pattern match
 packages_with_system_tests_pattern=$(printf "|*%s*" "${packages_with_system_tests[@]}")
 packages_with_system_tests_pattern="${packages_with_system_tests_pattern:1}" # Remove the leading pipe
 
-
-# Run system tests for each package with directory packages/*/tests/system
-for dir in `find 'packages' -type d -wholename 'packages/*/tests/system'`; do
-  # Get the path to the package by removing the suffix /tests/system
-  package=$(echo $dir | cut -f -2 -d '/')
-
-  # Run system tests on every change to these libraries
-  if [[ $package == @($packages_with_system_tests_pattern) ]]; then
-    files_to_check=${package}
-  else
-    files_to_check=${package}/CHANGELOG.md
+# Run system tests for each package with directory `packages/*/tests/system` or directory `packages/*/system_tests`
+for dir in `find 'packages' -type d -wholename 'packages/*/tests/system' -o -wholename 'packages/*/system_tests'`; do
+  # Extract the package name and define the relative package path
+  # 1. Remove the 'packages/' prefix from the start
+  # 2. Remove everything after the first '/' remaining
+  package_name=${dir#packages/}
+  package_name=${package_name%%/*}
+  package_path="packages/${package_name}"
+
+  # Determine if we should skip based on git diff
+  files_to_check="${package_path}/CHANGELOG.md"
+  if [[ $package_name == @($packages_with_system_tests_pattern) ]]; then
+    files_to_check="${package_path}"
   fi
 
   echo "checking changes with 'git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- ${files_to_check}'"
   set +e
   package_modified=$(git diff "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}" -- ${files_to_check} | wc -l)
   set -e
-  if [[ "${package_modified}" -eq 0 ]]; then
-      echo "no change detected in ${files_to_check}, skipping"
+
+  if [[ "${package_modified}" -gt 0 ]]; then
+      # Call the function - its internal exports won't affect the next loop
+      run_package_test "$package_name" || RETVAL=$?
   else
-      echo "change detected in ${files_to_check}"
-      echo "Running system tests for ${package}"
-      pushd ${package}
-      # Temporarily allow failure.
-      set +e
-      ${system_test_script}
-      ret=$?
-      set -e
-      if [ ${ret} -ne 0 ]; then
-          RETVAL=${ret}
-      fi
-      popd
+      echo "No changes in ${package_name}, skipping."
   fi
 done
 exit ${RETVAL}

.librarian/generator-input/packages/google-cloud-hypercomputecluster/.repo-metadata.json

@@ -0,0 +1,16 @@
+{
+  "api_description": "The Cluster Director API allows you to deploy, manage, and monitor clusters that run AI, ML, or HPC workloads.",
+  "api_id": "hypercomputecluster.googleapis.com",
+  "api_shortname": "hypercomputecluster",
+  "client_documentation": "https://cloud.google.com/python/docs/reference/google-cloud-hypercomputecluster/latest",
+  "default_version": "v1",
+  "distribution_name": "google-cloud-hypercomputecluster",
+  "issue_tracker": "https://issuetracker.google.com/issues/new?component=1907878&template=2195617",
+  "language": "python",
+  "library_type": "GAPIC_AUTO",
+  "name": "google-cloud-hypercomputecluster",
+  "name_pretty": "Cluster Director API",
+  "product_documentation": "https://cloud.google.com/blog/products/compute/managed-slurm-and-other-cluster-director-enhancements",
+  "release_level": "preview",
+  "repo": "googleapis/google-cloud-python"
+}

.librarian/state.yaml

@@ -9,6 +9,15 @@ libraries:
     preserve_regex: []
     remove_regex: []
     tag_format: '{id}-v{version}'
+  - id: db-dtypes
+    version: 1.5.0
+    last_generated_commit: ""
+    apis: []
+    source_roots:
+      - packages/db-dtypes
+    preserve_regex: []
+    remove_regex: []
+    tag_format: '{id}-v{version}'
   - id: google-ads-admanager
     version: 0.8.0
     last_generated_commit: effe5c4fa816021e724ca856d5640f2e55b14a8b
@@ -222,6 +231,15 @@ libraries:
     remove_regex:
       - packages/google-area120-tables/
     tag_format: '{id}-v{version}'
+  - id: google-auth
+    version: 2.49.0
+    last_generated_commit: ""
+    apis: []
+    source_roots:
+      - packages/google-auth
+    preserve_regex: []
+    remove_regex: []
+    tag_format: '{id}-v{version}'
   - id: google-auth-httplib2
     version: 0.3.0
     last_generated_commit: ""
@@ -287,8 +305,8 @@ libraries:
       - packages/google-cloud-advisorynotifications/
     tag_format: '{id}-v{version}'
   - id: google-cloud-alloydb
-    version: 0.7.0
-    last_generated_commit: d4a34bf03d617723146fe3ae15192c4d93981a27
+    version: 0.8.0
+    last_generated_commit: 5c9602dbb5ac6856c07daf83be1fbd001a972ef3
     apis:
       - path: google/cloud/alloydb/v1beta
         service_config: alloydb_v1beta.yaml
@@ -523,7 +541,7 @@ libraries:
     tag_format: '{id}-v{version}'
   - id: google-cloud-auditmanager
     version: 0.1.0
-    last_generated_commit: 9eea40c74d97622bb0aa406dd313409a376cc73b
+    last_generated_commit: 5c9602dbb5ac6856c07daf83be1fbd001a972ef3
     apis:
       - path: google/cloud/auditmanager/v1
         service_config: auditmanager_v1.yaml
@@ -1041,7 +1059,7 @@ libraries:
     tag_format: '{id}-v{version}'
   - id: google-cloud-commerce-consumer-procurement
     version: 0.4.0
-    last_generated_commit: 3322511885371d2b2253f209ccc3aa60d4100cfd
+    last_generated_commit: 5c9602dbb5ac6856c07daf83be1fbd001a972ef3
     apis:
       - path: google/cloud/commerce/consumer/procurement/v1
         service_config: cloudcommerceconsumerprocurement_v1.yaml
@@ -1071,8 +1089,8 @@ libraries:
       - packages/google-cloud-common/
     tag_format: '{id}-v{version}'
   - id: google-cloud-compute
-    version: 1.44.0
-    last_generated_commit: 69bf99a9d0910fa86c3c5867ea357aa004dfcc55
+    version: 1.45.0
+    last_generated_commit: e8a3d84236cc27875073b1f817e49d7faee11f0d
     apis:
       - path: google/cloud/compute/v1
         service_config: compute_v1.yaml
@@ -1086,8 +1104,8 @@ libraries:
       - packages/google-cloud-compute/
     tag_format: '{id}-v{version}'
   - id: google-cloud-compute-v1beta
-    version: 0.7.0
-    last_generated_commit: 69bf99a9d0910fa86c3c5867ea357aa004dfcc55
+    version: 0.8.0
+    last_generated_commit: e8a3d84236cc27875073b1f817e49d7faee11f0d
     apis:
       - path: google/cloud/compute/v1beta
         service_config: compute_v1beta.yaml
@@ -1586,8 +1604,8 @@ libraries:
     remove_regex: []
     tag_format: '{id}-v{version}'
   - id: google-cloud-documentai
-    version: 3.10.0
-    last_generated_commit: 9eea40c74d97622bb0aa406dd313409a376cc73b
+    version: 3.11.0
+    last_generated_commit: 5c9602dbb5ac6856c07daf83be1fbd001a972ef3
     apis:
       - path: google/cloud/documentai/v1beta3
         service_config: documentai_v1beta3.yaml
@@ -1601,6 +1619,15 @@ libraries:
     remove_regex:
       - packages/google-cloud-documentai/
     tag_format: '{id}-v{version}'
+  - id: google-cloud-documentai-toolbox
+    version: 0.15.1-alpha
+    last_generated_commit: ""
+    apis: []
+    source_roots:
+      - packages/google-cloud-documentai-toolbox
+    preserve_regex: []
+    remove_regex: []
+    tag_format: '{id}-v{version}'
   - id: google-cloud-domains
     version: 1.12.0
     last_generated_commit: 3322511885371d2b2253f209ccc3aa60d4100cfd
@@ -1767,15 +1794,15 @@ libraries:
       - packages/google-cloud-financialservices/
     tag_format: '{id}-v{version}'
   - id: google-cloud-firestore
-    version: 2.23.0
-    last_generated_commit: d420134ab324c0cbe0f4ae06ad9697dac77f26ad
+    version: 2.24.0
+    last_generated_commit: 572d5d51926dfa9e7509e12ec61e98ce19650f7c
     apis:
-      - path: google/firestore/v1
-        service_config: firestore_v1.yaml
       - path: google/firestore/admin/v1
         service_config: firestore_v1.yaml
       - path: google/firestore/bundle
         service_config: ""
+      - path: google/firestore/v1
+        service_config: firestore_v1.yaml
     source_roots:
       - packages/google-cloud-firestore
     preserve_regex:
@@ -1969,11 +1996,13 @@ libraries:
       - packages/google-cloud-gsuiteaddons
     tag_format: '{id}-v{version}'
   - id: google-cloud-hypercomputecluster
-    version: 0.2.0
-    last_generated_commit: c662840a94dbdf708caa44893a2d49119cdd391c
+    version: 0.3.0
+    last_generated_commit: 23ec7b6ee94a2fe364cdb5a212949f1cf37f85ad
     apis:
       - path: google/cloud/hypercomputecluster/v1beta
         service_config: hypercomputecluster_v1beta.yaml
+      - path: google/cloud/hypercomputecluster/v1
+        service_config: hypercomputecluster_v1.yaml
     source_roots:
       - packages/google-cloud-hypercomputecluster
     preserve_regex:
@@ -2140,7 +2169,7 @@ libraries:
       - packages/google-cloud-locationfinder/
     tag_format: '{id}-v{version}'
   - id: google-cloud-logging
-    version: 3.13.0
+    version: 3.14.0
     last_generated_commit: 69bf99a9d0910fa86c3c5867ea357aa004dfcc55
     apis:
       - path: google/logging/v2
@@ -2412,8 +2441,8 @@ libraries:
       - packages/google-cloud-network-connectivity/
     tag_format: '{id}-v{version}'
   - id: google-cloud-network-management
-    version: 1.32.0
-    last_generated_commit: cfe62b3c5ceb9f81879c61480f26707980534462
+    version: 1.33.0
+    last_generated_commit: 5c9602dbb5ac6856c07daf83be1fbd001a972ef3
     apis:
       - path: google/cloud/networkmanagement/v1
         service_config: networkmanagement_v1.yaml
@@ -3128,8 +3157,8 @@ libraries:
       - packages/google-cloud-storage-transfer/
     tag_format: '{id}-v{version}'
   - id: google-cloud-storagebatchoperations
-    version: 0.4.0
-    last_generated_commit: 1133adb136f742df62864f1d9d307df25d451880
+    version: 0.5.0
+    last_generated_commit: 5c9602dbb5ac6856c07daf83be1fbd001a972ef3
     apis:
       - path: google/cloud/storagebatchoperations/v1
         service_config: storagebatchoperations_v1.yaml
@@ -3306,8 +3335,8 @@ libraries:
       - packages/google-cloud-translate/
     tag_format: '{id}-v{version}'
   - id: google-cloud-vectorsearch
-    version: 0.5.0
-    last_generated_commit: 45e46bdae3874d2b92a2ced22529fc7037a1878f
+    version: 0.6.0
+    last_generated_commit: 5c9602dbb5ac6856c07daf83be1fbd001a972ef3
     apis:
       - path: google/cloud/vectorsearch/v1beta
         service_config: vectorsearch_v1beta.yaml
@@ -3951,8 +3980,8 @@ libraries:
       - packages/google-shopping-type/
     tag_format: '{id}-v{version}'
   - id: googleapis-common-protos
-    version: 1.72.0
-    last_generated_commit: c662840a94dbdf708caa44893a2d49119cdd391c
+    version: 1.73.0
+    last_generated_commit: 5c9602dbb5ac6856c07daf83be1fbd001a972ef3
     apis:
       - path: google/api
         service_config: serviceconfig.yaml