Add self-validating sandbox example E2E tests

.github/workflows/examples-e2e.yml

@@ -0,0 +1,71 @@
+name: Sandbox examples E2E
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 6 * * 1"
+
+concurrency:
+  group: sandbox-examples-e2e
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  examples:
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        include:
+          - name: synchronous-core
+            runner: 00_run_all.py
+            flows: 01_,02_,03_,04_,05_,06_,07_,08_,09_,10_,11_,12_,13_,14_,15_,20_,21_,22_
+            timeout: 900
+          - name: asynchronous
+            runner: 00_run_all_async.py
+            flows: all
+            timeout: 1800
+          - name: lifecycle
+            runner: 00_run_all.py
+            flows: 16_,17_,18_,19_
+            timeout: 900
+          - name: snapshots
+            runner: 00_run_all.py
+            flows: 23_,24_,25_
+            timeout: 1800
+    name: ${{ matrix.name }}
+    env:
+      KOYEB_API_TOKEN: ${{ secrets.KOYEB_API_TOKEN }}
+      KOYEB_API_HOST: ${{ vars.KOYEB_API_HOST || 'https://app.koyeb.com' }}
+      KOYEB_PROJECT_ID: ${{ vars.KOYEB_PROJECT_ID }}
+      KOYEB_REGION: ${{ vars.KOYEB_REGION || 'na' }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - run: python -m pip install --upgrade uv
+      - run: uv sync --frozen
+      - run: uv run python examples/${{ matrix.runner }} --flows "${{ matrix.flows }}" --timeout "${{ matrix.timeout }}"
+
+  snapshot-benchmark:
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    env:
+      KOYEB_API_TOKEN: ${{ secrets.KOYEB_API_TOKEN }}
+      KOYEB_API_HOST: ${{ vars.KOYEB_API_HOST || 'https://app.koyeb.com' }}
+      KOYEB_PROJECT_ID: ${{ vars.KOYEB_PROJECT_ID }}
+      KOYEB_REGION: ${{ vars.KOYEB_REGION || 'na' }}
+      KOYEB_SNAPSHOT_BENCHMARK_INSTANCE_TYPE: micro
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - run: python -m pip install --upgrade uv
+      - run: uv sync --frozen
+      - run: uv run python examples/26_snapshot_boot_benchmark.py --fs-sizes 1 --full-sizes 1 --boots 1 --csv /tmp/snapshot-boot.csv

examples/00_run_all.py

@@ -44,12 +44,19 @@ def main():
         ]
     )
 
+    expected_numbers = {f"{number:02d}" for number in range(1, 27)}
+    present_numbers = {example.name[:2] for example in all_example_files}
+    missing_numbers = sorted(expected_numbers - present_numbers)
+    if missing_numbers:
+        print(f"Missing synchronous example scenarios: {', '.join(missing_numbers)}")
+        return 1
+
     # Filter flows based on specification
     example_files = filter_flows(all_example_files, args.flows)
 
     if not example_files:
         print("No example files match the specified flows")
-        return 0
+        return 1
 
     # Build flow timeout mapping
     flow_timeouts = build_flow_timeouts(args.flow_timeout)
@@ -120,7 +127,7 @@ def main():
                     "name": example_name,
                     "status": "TIMEOUT",
                     "time": elapsed_time,
-                    "error": "Script exceeded 60 second timeout",
+                    "error": f"Script exceeded {timeout} second timeout",
                 }
             )
 

examples/00_run_all_async.py

@@ -105,7 +105,7 @@ async def run_example(example_file, timeout):
                 "name": example_name,
                 "status": "TIMEOUT",
                 "time": elapsed_time,
-                "error": "Script exceeded 60 second timeout"
+                "error": f"Script exceeded {timeout} second timeout"
             }
 
         elapsed_time = time.time() - start_time
@@ -183,7 +183,7 @@ async def main():
 
     if not example_files:
         print("No async example files match the specified flows")
-        return 0
+        return 1
 
     # Build flow timeout mapping
     flow_timeouts = build_flow_timeouts(args.flow_timeout)

examples/01_create_sandbox.py

@@ -28,10 +28,13 @@ def main():
         # Check health
         is_healthy = sandbox.is_healthy()
         print(f"Healthy: {is_healthy}")
+        assert is_healthy, "Sandbox should be healthy"
 
         # Test command
         result = sandbox.exec("echo 'Sandbox is ready!'")
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert result.stdout.strip() == "Sandbox is ready!"
 
         return 0
     finally:

examples/01_create_sandbox_async.py

@@ -30,10 +30,13 @@ async def main():
         # Check health
         is_healthy = await sandbox.is_healthy()
         print(f"Healthy: {is_healthy}")
+        assert is_healthy, "Sandbox should be healthy"
 
         # Test command
         result = await sandbox.exec("echo 'Sandbox is ready!'")
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert result.stdout.strip() == "Sandbox is ready!"
 
         return 0
     finally:

examples/02_create_sandbox_with_timing.py

@@ -100,6 +100,7 @@ def main(run_long_tests=False):
         health_duration = time.time() - health_start
         tracker.record("Health check", health_duration, "monitoring")
         print(f"    ✓ took {health_duration:.1f}s")
+        assert is_healthy, "Sandbox should be healthy"
 
         # Test command execution with timing
         print("  → Executing initial test command...")
@@ -108,6 +109,8 @@ def main(run_long_tests=False):
         exec_duration = time.time() - exec_start
         tracker.record("Initial exec command", exec_duration, "execution")
         print(f"    ✓ took {exec_duration:.1f}s")
+        assert result.exit_code == 0, result.stderr
+        assert result.stdout.strip() == "Sandbox is ready!"
 
         if run_long_tests:
             # Long test 1: Install a package
@@ -117,6 +120,7 @@ def main(run_long_tests=False):
             install_duration = time.time() - install_start
             tracker.record("Package installation", install_duration, "long_tests")
             print(f"    ✓ took {install_duration:.1f}s")
+            assert result.exit_code == 0, result.stderr
 
             # Long test 2: Run a computation
             print("  → [LONG TEST] Running computation...")
@@ -125,6 +129,7 @@ def main(run_long_tests=False):
             compute_duration = time.time() - compute_start
             tracker.record("Heavy computation", compute_duration, "long_tests")
             print(f"    ✓ took {compute_duration:.1f}s")
+            assert result.exit_code == 0, result.stderr
 
             # Long test 3: Multiple health checks
             print("  → [LONG TEST] Multiple health checks...")

examples/02_create_sandbox_with_timing_async.py

@@ -101,37 +101,42 @@ async def main(run_long_tests=False):
         # Check health with timing
         print("  → Checking sandbox health...")
         health_start = time.time()
-        await sandbox.is_healthy()
+        is_healthy = await sandbox.is_healthy()
         health_duration = time.time() - health_start
         tracker.record("Health check", health_duration, "monitoring")
         print(f"    ✓ took {health_duration:.1f}s")
+        assert is_healthy, "Sandbox should be healthy"
 
         # Test command execution with timing
         print("  → Executing initial test command...")
         exec_start = time.time()
-        await sandbox.exec("echo 'Sandbox is ready!'")
+        result = await sandbox.exec("echo 'Sandbox is ready!'")
         exec_duration = time.time() - exec_start
         tracker.record("Initial exec command", exec_duration, "execution")
         print(f"    ✓ took {exec_duration:.1f}s")
+        assert result.exit_code == 0, result.stderr
+        assert result.stdout.strip() == "Sandbox is ready!"
 
         if run_long_tests:
             # Long test 1: Install a package
             print("  → [LONG TEST] Installing a package...")
             install_start = time.time()
-            await sandbox.exec("pip install requests")
+            result = await sandbox.exec("pip install requests")
             install_duration = time.time() - install_start
             tracker.record("Package installation", install_duration, "long_tests")
             print(f"    ✓ took {install_duration:.1f}s")
+            assert result.exit_code == 0, result.stderr
 
             # Long test 2: Run a computation
             print("  → [LONG TEST] Running computation...")
             compute_start = time.time()
-            await sandbox.exec(
+            result = await sandbox.exec(
                 "python -c 'import time; sum(range(10000000)); time.sleep(2)'"
             )
             compute_duration = time.time() - compute_start
             tracker.record("Heavy computation", compute_duration, "long_tests")
             print(f"    ✓ took {compute_duration:.1f}s")
+            assert result.exit_code == 0, result.stderr
 
             # Long test 3: Multiple health checks
             print("  → [LONG TEST] Multiple health checks...")

examples/03_basic_commands.py

@@ -28,10 +28,14 @@ def main():
         # Simple command
         result = sandbox.exec("echo 'Hello World'")
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert result.stdout.strip() == "Hello World"
 
         # Python command
         result = sandbox.exec("python3 -c 'print(2 + 2)'")
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert result.stdout.strip() == "4"
 
         # Multi-line Python script
         result = sandbox.exec(
@@ -42,6 +46,9 @@ def main():
 "'''
         )
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert "Python version:" in result.stdout
+        assert "Platform:" in result.stdout
 
         # Failing command returns non-zero exit code
         result = sandbox.exec("ls /nonexistent")

examples/03_basic_commands_async.py

@@ -30,10 +30,14 @@ async def main():
         # Simple command
         result = await sandbox.exec("echo 'Hello World'")
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert result.stdout.strip() == "Hello World"
 
         # Python command
         result = await sandbox.exec("python3 -c 'print(2 + 2)'")
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert result.stdout.strip() == "4"
 
         # Multi-line Python script
         result = await sandbox.exec(
@@ -44,6 +48,9 @@ async def main():
 "'''
         )
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert "Python version:" in result.stdout
+        assert "Platform:" in result.stdout
 
         return 0
     finally:

examples/04_streaming_output.py

@@ -38,6 +38,8 @@ def main():
             on_stderr=lambda data: print(f"ERR: {data.strip()}"),
         )
         print(f"\nExit code: {result.exit_code}")
+        assert result.exit_code == 0, result.stderr
+        assert "Line 5" in result.stdout
 
         # Stream a script
         sandbox.filesystem.write_file(
@@ -50,6 +52,8 @@ def main():
             "python3 /tmp/counter.py",
             on_stdout=lambda data: print(data.strip()),
         )
+        assert result.exit_code == 0, result.stderr
+        assert "Done!" in result.stdout
 
         # Failing command with streaming returns non-zero exit code
         result = sandbox.exec(

examples/04_streaming_output_async.py

@@ -40,6 +40,8 @@ async def main():
             on_stderr=lambda data: print(f"ERR: {data.strip()}"),
         )
         print(f"\nExit code: {result.exit_code}")
+        assert result.exit_code == 0, result.stderr
+        assert "Line 5" in result.stdout
 
         # Stream a script
         await sandbox.filesystem.write_file(
@@ -52,6 +54,8 @@ async def main():
             "python3 /tmp/counter.py",
             on_stdout=lambda data: print(data.strip()),
         )
+        assert result.exit_code == 0, result.stderr
+        assert "Done!" in result.stdout
 
         return 0
     finally:

examples/05_environment_variables.py

@@ -28,7 +28,11 @@ def main():
     try:
         # Create a secret
         secret_response = secrets_api.create_secret(
-            secret=CreateSecret(name=secret_name, value=secret_value)
+            secret=CreateSecret(
+                name=secret_name,
+                value=secret_value,
+                project_id=os.getenv("KOYEB_PROJECT_ID") or None,
+            )
         )
         secret_id = secret_response.secret.id
         print(f"Created secret: {secret_name}")

examples/05_environment_variables_async.py

@@ -29,7 +29,11 @@ async def main():
     try:
         # Create a secret
         secret_response = secrets_api.create_secret(
-            secret=CreateSecret(name=secret_name, value=secret_value)
+            secret=CreateSecret(
+                name=secret_name,
+                value=secret_value,
+                project_id=os.getenv("KOYEB_PROJECT_ID") or None,
+            )
         )
         secret_id = secret_response.secret.id
         print(f"Created secret: {secret_name}")

examples/06_working_directory.py

@@ -33,14 +33,20 @@ def main():
         # Run command in specific directory
         result = sandbox.exec("pwd", cwd="/tmp/my_project")
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert result.stdout.strip() == "/tmp/my_project"
 
         # List files in working directory
         result = sandbox.exec("ls -la", cwd="/tmp/my_project")
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert "src" in result.stdout
 
         # Use relative paths
         result = sandbox.exec("cat src/main.py", cwd="/tmp/my_project")
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert 'print("hello")' in result.stdout
 
         return 0
 

examples/06_working_directory_async.py

@@ -35,14 +35,20 @@ async def main():
         # Run command in specific directory
         result = await sandbox.exec("pwd", cwd="/tmp/my_project")
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert result.stdout.strip() == "/tmp/my_project"
 
         # List files in working directory
         result = await sandbox.exec("ls -la", cwd="/tmp/my_project")
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert "src" in result.stdout
 
         # Use relative paths
         result = await sandbox.exec("cat src/main.py", cwd="/tmp/my_project")
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert 'print("hello")' in result.stdout
 
         return 0
 

examples/07_file_operations.py

@@ -35,13 +35,16 @@ def main():
         # Read file
         file_info = fs.read_file("/tmp/hello.txt")
         print(file_info.content)
+        assert file_info.content == content
 
         # Write Python script
         python_code = "#!/usr/bin/env python3\nprint('Hello from Python!')\n"
         fs.write_file("/tmp/script.py", python_code)
         sandbox.exec("chmod +x /tmp/script.py")
         result = sandbox.exec("/tmp/script.py")
         print(result.stdout.strip())
+        assert result.exit_code == 0, result.stderr
+        assert result.stdout.strip() == "Hello from Python!"
 
         return 0