test: run Nexus tests against Cloud

.github/scripts/cloud_namespace.py

@@ -51,8 +51,10 @@ async def wait_for_operation(
 
 async def create() -> None:
     client = await cloud_client()
-    namespace_name = "sdk-python-ci-{}-{}".format(
-        os.environ["GITHUB_RUN_ID"], os.environ["GITHUB_RUN_ATTEMPT"]
+    namespace_name = "sdk-python-ci-{}-{}{}".format(
+        os.environ["GITHUB_RUN_ID"],
+        os.environ["GITHUB_RUN_ATTEMPT"],
+        os.environ.get("TEMPORAL_CLOUD_NAMESPACE_SUFFIX", ""),
     )
     result = await client.cloud_service.create_namespace(
         CreateNamespaceRequest(

.github/workflows/ci.yml

@@ -254,8 +254,9 @@ jobs:
         env:
           TEMPORAL_CLIENT_CLOUD_API_KEY: ${{ secrets.TEMPORAL_CLIENT_CLOUD_API_KEY }}
           TEMPORAL_CLIENT_CLOUD_API_VERSION: v0.19.1
+          TEMPORAL_CLOUD_NAMESPACE_SUFFIX: -general
       - run: mkdir junit-xml
-      - run: poe test -s --workflow-environment envconfig --junit-xml=junit-xml/cloud.xml
+      - run: poe test -s --workflow-environment envconfig --ignore=tests/nexus --junit-xml=junit-xml/cloud.xml
         timeout-minutes: 15
         env:
           TEMPORAL_ADDRESS: ${{ steps.create-cloud-namespace.outputs.namespace }}.tmprl.cloud:7233
@@ -278,6 +279,80 @@ jobs:
           path: junit-xml
           retention-days: 14
 
+  # Nexus endpoint provisioning is slow on Cloud, so run these tests separately.
+  cloud-nexus-test:
+    if: ${{ github.event.pull_request.head.repo.full_name == '' || github.event.pull_request.head.repo.full_name == 'temporalio/sdk-python' }}
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          submodules: recursive
+      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.14"
+      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+        with:
+          workspaces: temporalio/bridge -> target
+          key: ${{ env.pythonLocation }}
+      - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3
+        with:
+          version: "23.x"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8
+      - run: uv tool install poethepoet
+      - run: uv sync --all-extras
+      - run: poe build-develop
+      - name: Generate Cloud test certificates
+        run: |
+          cert_dir="$RUNNER_TEMP/cloud-test-certs"
+          mkdir "$cert_dir"
+          openssl req -x509 -newkey rsa:2048 -nodes -days 1 \
+            -keyout "$cert_dir/ca.key" -out "$cert_dir/ca.pem" \
+            -subj '/CN=Temporal Python SDK Cloud CI CA'
+          openssl req -newkey rsa:2048 -nodes \
+            -keyout "$cert_dir/client.key" -out "$cert_dir/client.csr" \
+            -subj '/CN=Temporal Python SDK Cloud CI'
+          openssl x509 -req -days 1 -in "$cert_dir/client.csr" \
+            -CA "$cert_dir/ca.pem" -CAkey "$cert_dir/ca.key" -CAcreateserial \
+            -out "$cert_dir/client.pem" -extfile <(printf 'extendedKeyUsage=clientAuth')
+          {
+            echo "TEMPORAL_CLOUD_CLIENT_CA_PATH=$cert_dir/ca.pem"
+            echo "TEMPORAL_TLS_CLIENT_CERT_PATH=$cert_dir/client.pem"
+            echo "TEMPORAL_TLS_CLIENT_KEY_PATH=$cert_dir/client.key"
+          } >> "$GITHUB_ENV"
+      - name: Create Cloud namespace
+        id: create-cloud-namespace
+        run: uv run python .github/scripts/cloud_namespace.py create
+        env:
+          TEMPORAL_CLIENT_CLOUD_API_KEY: ${{ secrets.TEMPORAL_CLIENT_CLOUD_API_KEY }}
+          TEMPORAL_CLIENT_CLOUD_API_VERSION: v0.19.1
+          TEMPORAL_CLOUD_NAMESPACE_SUFFIX: -nexus
+      - run: mkdir junit-xml
+      - run: poe test -n 16 -s --workflow-environment envconfig tests/nexus --junit-xml=junit-xml/cloud-nexus.xml
+        timeout-minutes: 45
+        env:
+          TEMPORAL_ADDRESS: ${{ steps.create-cloud-namespace.outputs.namespace }}.tmprl.cloud:7233
+          TEMPORAL_NAMESPACE: ${{ steps.create-cloud-namespace.outputs.namespace }}
+          TEMPORAL_IS_CLOUD_TESTS: true
+          TEMPORAL_CLIENT_CLOUD_API_KEY: ${{ secrets.TEMPORAL_CLIENT_CLOUD_API_KEY }}
+          TEMPORAL_CLIENT_CLOUD_API_VERSION: v0.19.1
+          TEMPORAL_CLIENT_CLOUD_NAMESPACE: ${{ steps.create-cloud-namespace.outputs.namespace }}
+      - name: Delete Cloud namespace
+        if: ${{ always() && steps.create-cloud-namespace.outputs.namespace != '' }}
+        run: uv run python .github/scripts/cloud_namespace.py delete "${{ steps.create-cloud-namespace.outputs.namespace }}"
+        env:
+          TEMPORAL_CLIENT_CLOUD_API_KEY: ${{ secrets.TEMPORAL_CLIENT_CLOUD_API_KEY }}
+          TEMPORAL_CLIENT_CLOUD_API_VERSION: v0.19.1
+      - name: Upload junit-xml artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        if: always()
+        with:
+          name: junit-xml--${{github.run_id}}--${{github.run_attempt}}--cloud-nexus
+          path: junit-xml
+          retention-days: 14
+
   # Runs the sdk features repo tests with this repo's current SDK code
   features-tests:
     uses: temporalio/features/.github/workflows/python.yaml@main

tests/nexus/conftest.py

@@ -0,0 +1,193 @@
+import asyncio
+import os
+import time
+from collections.abc import AsyncGenerator
+from dataclasses import dataclass
+
+import pytest
+import pytest_asyncio
+
+from temporalio.api.cloud.cloudservice.v1 import (
+    CreateNexusEndpointRequest,
+    DeleteNexusEndpointRequest,
+    GetAsyncOperationRequest,
+    GetNamespaceRequest,
+    GetNexusEndpointRequest,
+)
+from temporalio.api.cloud.nexus.v1 import (
+    AllowedCloudNamespacePolicySpec,
+    Endpoint,
+    EndpointPolicySpec,
+    EndpointSpec,
+    EndpointTargetSpec,
+    WorkerTargetSpec,
+)
+from temporalio.api.cloud.operation.v1 import AsyncOperation
+from temporalio.api.cloud.resource.v1 import ResourceState
+from temporalio.api.operatorservice.v1 import (
+    GetNexusEndpointRequest as GetDataPlaneNexusEndpointRequest,
+)
+from temporalio.client import CloudOperationsClient
+from temporalio.service import RPCError, RPCStatusCode
+from temporalio.testing import WorkflowEnvironment
+
+
+@dataclass
+class _CloudNexusEndpointClient:
+    client: CloudOperationsClient
+    namespace_id: str
+
+    async def wait_for_operation(self, operation: AsyncOperation) -> None:
+        deadline = time.monotonic() + 10 * 60
+        while True:
+            operation = (
+                await self.client.cloud_service.get_async_operation(
+                    GetAsyncOperationRequest(async_operation_id=operation.id)
+                )
+            ).async_operation
+            if operation.state == AsyncOperation.STATE_FULFILLED:
+                return
+            if operation.state in {
+                AsyncOperation.STATE_FAILED,
+                AsyncOperation.STATE_CANCELLED,
+                AsyncOperation.STATE_REJECTED,
+            }:
+                raise RuntimeError(
+                    "Cloud operation "
+                    f"{operation.id} "
+                    f"{AsyncOperation.State.Name(operation.state).lower()}: "
+                    f"{operation.failure_reason}"
+                )
+            if time.monotonic() >= deadline:
+                raise TimeoutError(
+                    f"Timed out waiting for Cloud operation {operation.id}"
+                )
+            delay = max(
+                operation.check_duration.seconds
+                + operation.check_duration.nanos / 1_000_000_000,
+                1,
+            )
+            await asyncio.sleep(min(delay, deadline - time.monotonic()))
+
+    async def wait_for_endpoint(self, endpoint_id: str) -> Endpoint:
+        deadline = time.monotonic() + 10 * 60
+        while True:
+            endpoint = (
+                await self.client.cloud_service.get_nexus_endpoint(
+                    GetNexusEndpointRequest(endpoint_id=endpoint_id)
+                )
+            ).endpoint
+            if endpoint.state == ResourceState.RESOURCE_STATE_ACTIVE:
+                return endpoint
+            if endpoint.state in {
+                ResourceState.RESOURCE_STATE_ACTIVATION_FAILED,
+                ResourceState.RESOURCE_STATE_UPDATE_FAILED,
+                ResourceState.RESOURCE_STATE_DELETE_FAILED,
+                ResourceState.RESOURCE_STATE_SUSPENDED,
+                ResourceState.RESOURCE_STATE_EXPIRED,
+            }:
+                raise RuntimeError(
+                    "Cloud Nexus endpoint "
+                    f"{endpoint_id} "
+                    f"{ResourceState.Name(endpoint.state).lower()}"
+                )
+            if time.monotonic() >= deadline:
+                raise TimeoutError(
+                    f"Timed out waiting for Cloud Nexus endpoint {endpoint_id}"
+                )
+            await asyncio.sleep(1)
+
+    async def wait_for_data_plane_endpoint(
+        self, env: WorkflowEnvironment, endpoint_id: str
+    ) -> None:
+        deadline = time.monotonic() + 10 * 60
+        while True:
+            try:
+                await env.client.operator_service.get_nexus_endpoint(
+                    GetDataPlaneNexusEndpointRequest(id=endpoint_id)
+                )
+                return
+            except RPCError as err:
+                if err.status != RPCStatusCode.NOT_FOUND:
+                    raise
+            if time.monotonic() >= deadline:
+                raise TimeoutError(
+                    "Timed out waiting for Cloud Nexus endpoint "
+                    f"{endpoint_id} to reach the data plane"
+                )
+            await asyncio.sleep(1)
+
+
+@pytest_asyncio.fixture(scope="session")  # type: ignore[reportUntypedFunctionDecorator]
+async def cloud_nexus_endpoint_client() -> AsyncGenerator[
+    _CloudNexusEndpointClient | None, None
+]:
+    if "TEMPORAL_IS_CLOUD_TESTS" not in os.environ:
+        yield None
+        return
+
+    client = await CloudOperationsClient.connect(
+        api_key=os.environ["TEMPORAL_CLIENT_CLOUD_API_KEY"],
+        version=os.environ["TEMPORAL_CLIENT_CLOUD_API_VERSION"],
+    )
+    namespace = await client.cloud_service.get_namespace(
+        GetNamespaceRequest(namespace=os.environ["TEMPORAL_NAMESPACE"])
+    )
+    yield _CloudNexusEndpointClient(client, namespace.namespace.namespace)
+
+
+@pytest_asyncio.fixture(autouse=True)  # type: ignore[reportUntypedFunctionDecorator]
+async def cloud_nexus_endpoints(
+    cloud_nexus_endpoint_client: _CloudNexusEndpointClient | None,
+    env: WorkflowEnvironment,
+    monkeypatch: pytest.MonkeyPatch,
+) -> AsyncGenerator[None, None]:
+    if cloud_nexus_endpoint_client is None:
+        yield
+        return
+
+    endpoints: list[Endpoint] = []
+
+    async def create_nexus_endpoint(endpoint_name: str, task_queue: str) -> Endpoint:
+        response = await cloud_nexus_endpoint_client.client.cloud_service.create_nexus_endpoint(
+            CreateNexusEndpointRequest(
+                spec=EndpointSpec(
+                    name=endpoint_name,
+                    target_spec=EndpointTargetSpec(
+                        worker_target_spec=WorkerTargetSpec(
+                            namespace_id=cloud_nexus_endpoint_client.namespace_id,
+                            task_queue=task_queue,
+                        )
+                    ),
+                    policy_specs=[
+                        EndpointPolicySpec(
+                            allowed_cloud_namespace_policy_spec=AllowedCloudNamespacePolicySpec(
+                                namespace_id=cloud_nexus_endpoint_client.namespace_id
+                            )
+                        )
+                    ],
+                )
+            )
+        )
+        await cloud_nexus_endpoint_client.wait_for_operation(response.async_operation)
+        endpoint = await cloud_nexus_endpoint_client.wait_for_endpoint(
+            response.endpoint_id
+        )
+        await cloud_nexus_endpoint_client.wait_for_data_plane_endpoint(env, endpoint.id)
+        endpoints.append(endpoint)
+        return endpoint
+
+    monkeypatch.setattr(env, "create_nexus_endpoint", create_nexus_endpoint)
+    try:
+        yield
+    finally:
+        for endpoint in reversed(endpoints):
+            response = await cloud_nexus_endpoint_client.client.cloud_service.delete_nexus_endpoint(
+                DeleteNexusEndpointRequest(
+                    endpoint_id=endpoint.id,
+                    resource_version=endpoint.resource_version,
+                )
+            )
+            await cloud_nexus_endpoint_client.wait_for_operation(
+                response.async_operation
+            )

tests/nexus/test_dynamic_creation_of_user_handler_classes.py

@@ -10,10 +10,6 @@
 from temporalio.worker import Worker
 from tests.helpers.nexus import make_nexus_endpoint_name
 
-# Cloud CI's namespace credentials cannot manage Nexus endpoints.
-# See https://github.com/temporalio/sdk-python/issues/1704.
-pytestmark = pytest.mark.requires_local_server
-
 
 @workflow.defn
 class MyWorkflow:

tests/nexus/test_nexus_client_updates.py

@@ -3,7 +3,6 @@
 import uuid
 
 import nexusrpc
-import pytest
 from nexusrpc.handler import StartOperationContext, service_handler, sync_operation
 
 import temporalio.nexus
@@ -12,10 +11,6 @@
 from temporalio.testing import WorkflowEnvironment
 from temporalio.worker import Worker
 
-# Cloud CI's namespace credentials cannot manage Nexus endpoints.
-# See https://github.com/temporalio/sdk-python/issues/1704.
-pytestmark = pytest.mark.requires_local_server
-
 
 @nexusrpc.service
 class ClientTestService:

tests/nexus/test_nexus_worker_shutdown.py

@@ -23,10 +23,6 @@
     make_nexus_endpoint_name,
 )
 
-# Cloud CI's namespace credentials cannot manage Nexus endpoints.
-# See https://github.com/temporalio/sdk-python/issues/1704.
-pytestmark = pytest.mark.requires_local_server
-
 
 @nexusrpc.service
 class ShutdownTestService:

tests/nexus/test_signal_link_propagation_e2e.py

@@ -56,10 +56,6 @@
     workflow_event_link_event_type,
 )
 
-# Cloud CI's namespace credentials cannot manage Nexus endpoints.
-# See https://github.com/temporalio/sdk-python/issues/1704.
-pytestmark = pytest.mark.requires_local_server
-
 EventType = temporalio.api.enums.v1.EventType
 
 

tests/nexus/test_standalone_operations.py

@@ -71,11 +71,6 @@
 # ---------------------------------------------------------------------------
 
 
-# Cloud CI's namespace credentials cannot manage Nexus endpoints.
-# See https://github.com/temporalio/sdk-python/issues/1704.
-pytestmark = pytest.mark.requires_local_server
-
-
 @dataclass
 class EchoInput:
     value: str

tests/nexus/test_temporal_extstore.py

@@ -41,10 +41,6 @@
 from tests.helpers.nexus import make_nexus_endpoint_name
 from tests.test_extstore import InMemoryTestDriver
 
-# Cloud CI's namespace credentials cannot manage Nexus endpoints.
-# See https://github.com/temporalio/sdk-python/issues/1704.
-pytestmark = pytest.mark.requires_local_server
-
 PAYLOAD_SIZE = 4096
 PAYLOAD_SIZE_THRESHOLD = 1024
 _STORE_FAILURE_MESSAGE = "external storage store failed"

tests/nexus/test_temporal_operation.py

@@ -40,10 +40,6 @@
     make_nexus_endpoint_name,
 )
 
-# Cloud CI's namespace credentials cannot manage Nexus endpoints.
-# See https://github.com/temporalio/sdk-python/issues/1704.
-pytestmark = pytest.mark.requires_local_server
-
 
 @dataclass
 class Input: