Release 0.1.38

kagglesdk/agents/services/agents_api_service.py

@@ -9,7 +9,7 @@ def __init__(self, client: KaggleHttpClient):
   def create_harness(self, request: ApiCreateHarnessRequest = None) -> ApiHarness:
     r"""
     Create a Harness together with its (required) first HarnessVersion.
-    Admin-only.
+    Requires the agentic-candidates feature flag.
 
     Example:
       curl -sSL -u andrewmingwang:local_api_token \
@@ -45,7 +45,7 @@ def create_harness(self, request: ApiCreateHarnessRequest = None) -> ApiHarness:
 
   def get_harness(self, request: ApiGetHarnessRequest = None) -> ApiHarness:
     r"""
-    Get a Harness by id. Admin-only.
+    Get a Harness by id. Requires the agentic-candidates feature flag.
 
     Example:
       curl -sSL -u andrewmingwang:local_api_token \
@@ -64,7 +64,7 @@ def get_harness(self, request: ApiGetHarnessRequest = None) -> ApiHarness:
   def create_harness_version(self, request: ApiCreateHarnessVersionRequest = None) -> ApiHarnessVersion:
     r"""
     Create an additional HarnessVersion under an existing Harness.
-    Admin-only.
+    Requires the agentic-candidates feature flag.
 
     Example:
       curl -sSL -u andrewmingwang:local_api_token \
@@ -95,7 +95,7 @@ def create_harness_version(self, request: ApiCreateHarnessVersionRequest = None)
 
   def get_harness_version(self, request: ApiGetHarnessVersionRequest = None) -> ApiHarnessVersion:
     r"""
-    Get a HarnessVersion by id. Admin-only.
+    Get a HarnessVersion by id. Requires the agentic-candidates feature flag.
 
     Example:
       curl -sSL -u andrewmingwang:local_api_token \
@@ -115,7 +115,8 @@ def list_harnesses(self, request: ApiListHarnessesRequest = None) -> ApiListHarn
     r"""
     List Harnesses. Returns one row per HarnessVersion visible to the caller
     (a Harness with N versions appears N times, each row with version
-    populated for that version). Admin-only.
+    populated for that version). Requires the agentic-candidates feature
+    flag.
 
     Example:
       curl -sSL -u andrewmingwang:local_api_token -G \
@@ -135,7 +136,8 @@ def list_harnesses(self, request: ApiListHarnessesRequest = None) -> ApiListHarn
   def create_agent(self, request: ApiCreateAgentRequest = None) -> ApiAgent:
     r"""
     Create an Agent (pairing of BenchmarkModelVersionConfig + HarnessVersion).
-    Admin-only, plus the caller must be able to read the referenced
+    Requires the agentic-candidates feature flag, plus the caller must be
+    able to read the referenced
     BenchmarkModelVersionConfig and HarnessVersion.
 
     Example:
@@ -164,7 +166,7 @@ def create_agent(self, request: ApiCreateAgentRequest = None) -> ApiAgent:
 
   def get_agent(self, request: ApiGetAgentRequest = None) -> ApiAgent:
     r"""
-    Get an Agent by id. Admin-only.
+    Get an Agent by id. Requires the agentic-candidates feature flag.
 
     Example:
       curl -sSL -u andrewmingwang:local_api_token \
@@ -182,7 +184,8 @@ def get_agent(self, request: ApiGetAgentRequest = None) -> ApiAgent:
 
   def list_agents(self, request: ApiListAgentsRequest = None) -> ApiListAgentsResponse:
     r"""
-    List Agents visible to the caller. Admin-only.
+    List Agents visible to the caller. Requires the agentic-candidates
+    feature flag.
 
     Example:
       curl -sSL -u andrewmingwang:local_api_token -G \

kagglesdk/agents/types/agent_service.py

@@ -10,12 +10,15 @@ class ListAgentsFilter(KaggleObject):
       Restrict to Agents whose BenchmarkModelVersionConfig has the given ids.
     harness_version_ids (int)
       Restrict to Agents whose HarnessVersion has the given ids.
+    slugs (str)
+      Restrict to Agents with the given slugs.
   """
 
   def __init__(self):
     self._agent_ids = []
     self._benchmark_model_version_config_ids = []
     self._harness_version_ids = []
+    self._slugs = []
     self._freeze()
 
   @property
@@ -66,6 +69,22 @@ def harness_version_ids(self, harness_version_ids: Optional[List[int]]):
       raise TypeError('harness_version_ids must contain only items of type int')
     self._harness_version_ids = harness_version_ids
 
+  @property
+  def slugs(self) -> Optional[List[str]]:
+    """Restrict to Agents with the given slugs."""
+    return self._slugs
+
+  @slugs.setter
+  def slugs(self, slugs: Optional[List[str]]):
+    if slugs is None:
+      del self.slugs
+      return
+    if not isinstance(slugs, list):
+      raise TypeError('slugs must be of type list')
+    if not all([isinstance(t, str) for t in slugs]):
+      raise TypeError('slugs must contain only items of type str')
+    self._slugs = slugs
+
 
 class ListHarnessesFilter(KaggleObject):
   r"""
@@ -122,6 +141,7 @@ def latest_version_per_harness_only(self, latest_version_per_harness_only: Optio
   FieldMetadata("agentIds", "agent_ids", "_agent_ids", int, [], ListSerializer(PredefinedSerializer())),
   FieldMetadata("benchmarkModelVersionConfigIds", "benchmark_model_version_config_ids", "_benchmark_model_version_config_ids", int, [], ListSerializer(PredefinedSerializer())),
   FieldMetadata("harnessVersionIds", "harness_version_ids", "_harness_version_ids", int, [], ListSerializer(PredefinedSerializer())),
+  FieldMetadata("slugs", "slugs", "_slugs", str, [], ListSerializer(PredefinedSerializer())),
 ]
 
 ListHarnessesFilter._fields = [

kagglesdk/benchmarks/services/benchmark_tasks_api_service.py

@@ -1,4 +1,5 @@
-from kagglesdk.benchmarks.types.benchmark_tasks_api_service import ApiBatchScheduleBenchmarkTaskRunsRequest, ApiBatchScheduleBenchmarkTaskRunsResponse, ApiBenchmarkTask, ApiCreateBenchmarkTaskRequest, ApiDownloadBenchmarkTaskRunOutputRequest, ApiGetBenchmarkTaskQuotaRequest, ApiGetBenchmarkTaskQuotaResponse, ApiGetBenchmarkTaskRequest, ApiGetBenchmarkTaskRunLogsRequest, ApiListBenchmarkTaskRunsRequest, ApiListBenchmarkTaskRunsResponse, ApiListBenchmarkTasksRequest, ApiListBenchmarkTasksResponse, ApiPublishBenchmarkTaskRequest, ApiUpdateBenchmarkTaskRequest, ApiUpdateBenchmarkTaskResponse, ApiUpdateBenchmarkTaskVersionRequest
+from kagglesdk.benchmarks.types.benchmark_task_run_service import GetBenchmarkSchedulerStatusResponse
+from kagglesdk.benchmarks.types.benchmark_tasks_api_service import ApiBatchScheduleBenchmarkTaskRunsRequest, ApiBatchScheduleBenchmarkTaskRunsResponse, ApiBenchmarkTask, ApiCancelBenchmarkTaskRunsRequest, ApiCancelBenchmarkTaskRunsResponse, ApiCreateBenchmarkTaskRequest, ApiDownloadBenchmarkTaskRunOutputRequest, ApiGetBenchmarkSchedulerStatusRequest, ApiGetBenchmarkTaskQuotaRequest, ApiGetBenchmarkTaskQuotaResponse, ApiGetBenchmarkTaskRequest, ApiGetBenchmarkTaskRunLogsRequest, ApiListBenchmarkTaskRunsRequest, ApiListBenchmarkTaskRunsResponse, ApiListBenchmarkTasksRequest, ApiListBenchmarkTasksResponse, ApiPublishBenchmarkTaskRequest, ApiUpdateBenchmarkTaskRequest, ApiUpdateBenchmarkTaskResponse, ApiUpdateBenchmarkTaskVersionRequest
 from kagglesdk.common.types.file_download import FileDownload
 from kagglesdk.kaggle_http_client import KaggleHttpClient
 
@@ -37,6 +38,22 @@ def batch_schedule_benchmark_task_runs(self, request: ApiBatchScheduleBenchmarkT
 
     return self._client.call("benchmarks.BenchmarkTasksApiService", "BatchScheduleBenchmarkTaskRuns", request, ApiBatchScheduleBenchmarkTaskRunsResponse)
 
+  def api_cancel_benchmark_task_runs(self, request: ApiCancelBenchmarkTaskRunsRequest = None) -> ApiCancelBenchmarkTaskRunsResponse:
+    r"""
+    Cancels the queued (and optionally the running) runs of a set of tasks,
+    optionally narrowed to one candidate. Only ever cancels the caller's own
+    runs; the flag-gated fleet-wide scope is only offered by the internal RPC.
+
+    Args:
+      request (ApiCancelBenchmarkTaskRunsRequest):
+        The request object; initialized to empty instance if not specified.
+    """
+
+    if request is None:
+      request = ApiCancelBenchmarkTaskRunsRequest()
+
+    return self._client.call("benchmarks.BenchmarkTasksApiService", "ApiCancelBenchmarkTaskRuns", request, ApiCancelBenchmarkTaskRunsResponse)
+
   def list_benchmark_tasks(self, request: ApiListBenchmarkTasksRequest = None) -> ApiListBenchmarkTasksResponse:
     r"""
     List tasks for the current user.
@@ -127,10 +144,28 @@ def get_benchmark_task_quota(self, request: ApiGetBenchmarkTaskQuotaRequest = No
 
     return self._client.call("benchmarks.BenchmarkTasksApiService", "GetBenchmarkTaskQuota", request, ApiGetBenchmarkTaskQuotaResponse)
 
+  def api_get_benchmark_scheduler_status(self, request: ApiGetBenchmarkSchedulerStatusRequest = None) -> GetBenchmarkSchedulerStatusResponse:
+    r"""
+    Scheduler readout covering the calling user's own enqueued runs: queue
+    depth, recent outcomes, and stats per candidate model and per benchmark.
+    Always self-scoped; the flag-gated fleet-wide view is only offered by the
+    internal RPC.
+
+    Args:
+      request (ApiGetBenchmarkSchedulerStatusRequest):
+        The request object; initialized to empty instance if not specified.
+    """
+
+    if request is None:
+      request = ApiGetBenchmarkSchedulerStatusRequest()
+
+    return self._client.call("benchmarks.BenchmarkTasksApiService", "ApiGetBenchmarkSchedulerStatus", request, GetBenchmarkSchedulerStatusResponse)
+
   def publish_benchmark_task(self, request: ApiPublishBenchmarkTaskRequest = None) -> ApiBenchmarkTask:
     r"""
     Publish a particular task for a user. Optionally publishes the backing
-    notebook (`source_kernel_id`) in the same request.
+    notebook (`source_kernel_id`) and the task's run artifacts in the same
+    request.
 
     Args:
       request (ApiPublishBenchmarkTaskRequest):

kagglesdk/benchmarks/services/benchmarks_api_service.py

@@ -1,4 +1,5 @@
-from kagglesdk.benchmarks.types.benchmarks_api_service import ApiAddTasksToBenchmarkRequest, ApiAddTasksToBenchmarkResponse, ApiBenchmarkLeaderboard, ApiBenchmarkModelVersionConfig, ApiCreateBenchmarkModelVersionConfigRequest, ApiCreateBenchmarkVersionAgentMappingsRequest, ApiCreateBenchmarkVersionAgentMappingsResponse, ApiDeleteBenchmarkVersionAgentMappingsRequest, ApiDeleteBenchmarkVersionAgentMappingsResponse, ApiGetBenchmarkLeaderboardRequest, ApiGetBenchmarkModelVersionConfigRequest, ApiListBenchmarkModelsRequest, ApiListBenchmarkModelsResponse, ApiListBenchmarkModelVersionConfigsRequest, ApiListBenchmarkModelVersionConfigsResponse, ApiListBenchmarkVersionAgentMappingsRequest, ApiListBenchmarkVersionAgentMappingsResponse
+from kagglesdk.benchmarks.types.benchmark_types import BenchmarkArenaSettings
+from kagglesdk.benchmarks.types.benchmarks_api_service import ApiAddTasksToBenchmarkRequest, ApiAddTasksToBenchmarkResponse, ApiBenchmarkLeaderboard, ApiBenchmarkModelVersionConfig, ApiCreateBenchmarkArenaSettingsRequest, ApiCreateBenchmarkModelVersionConfigRequest, ApiCreateBenchmarkVersionAgentMappingsRequest, ApiCreateBenchmarkVersionAgentMappingsResponse, ApiDeleteBenchmarkVersionAgentMappingsRequest, ApiDeleteBenchmarkVersionAgentMappingsResponse, ApiGetBenchmarkArenaSettingsRequest, ApiGetBenchmarkLeaderboardRequest, ApiGetBenchmarkModelVersionConfigRequest, ApiListBenchmarkArenaCombinationsRequest, ApiListBenchmarkArenaCombinationsResponse, ApiListBenchmarkModelsRequest, ApiListBenchmarkModelsResponse, ApiListBenchmarkModelVersionConfigsRequest, ApiListBenchmarkModelVersionConfigsResponse, ApiListBenchmarkVersionAgentMappingsRequest, ApiListBenchmarkVersionAgentMappingsResponse, ApiRegisterAuxiliaryModelVersionConfigsRequest, ApiRemoveTasksFromBenchmarkRequest, ApiRemoveTasksFromBenchmarkResponse, ApiUnregisterAuxiliaryModelVersionConfigsRequest, ApiUpdateBenchmarkArenaCombinationsRequest, ApiUpdateBenchmarkArenaSettingsRequest
 from kagglesdk.kaggle_http_client import KaggleHttpClient
 
 class BenchmarksApiClient(object):
@@ -189,11 +190,10 @@ def add_tasks_to_benchmark(self, request: ApiAddTasksToBenchmarkRequest = None)
     r"""
     Map one or more BenchmarkTasks into one of the current user's own
     benchmarks, making them part of that benchmark's task tree and
-    leaderboard. Only supported for benchmarks of type Personal. Requires
-    update access to the benchmark version and read access to each task. Tasks
-    already mapped to the benchmark are skipped and echoed back in
-    `skipped_tasks` rather than failing the request, so a partially-applied
-    bulk import can be safely re-run.
+    leaderboard. Requires update access to the benchmark version and read
+    access to each task. Tasks already mapped to the benchmark are skipped and
+    echoed back in `skipped_tasks` rather than failing the request, so a
+    partially-applied bulk import can be safely re-run.
 
     Example:
       curl -sSL -u fickleone:local_api_token \
@@ -216,3 +216,190 @@ def add_tasks_to_benchmark(self, request: ApiAddTasksToBenchmarkRequest = None)
       request = ApiAddTasksToBenchmarkRequest()
 
     return self._client.call("benchmarks.BenchmarksApiService", "AddTasksToBenchmark", request, ApiAddTasksToBenchmarkResponse)
+
+  def remove_tasks_from_benchmark(self, request: ApiRemoveTasksFromBenchmarkRequest = None) -> ApiRemoveTasksFromBenchmarkResponse:
+    r"""
+    Unmap one or more BenchmarkTasks from one of the current user's own
+    benchmarks, removing them from that benchmark's task tree and leaderboard.
+    Requires update access to the benchmark version and read access to each
+    task. Tasks not currently mapped to the benchmark are skipped and echoed
+    back in `skipped_tasks` rather than failing the request, so a
+    partially-applied bulk removal can be safely re-run.
+
+    Example:
+      curl -sSL -u fickleone:local_api_token \
+        -X POST \
+        http://localhost/api/v1/benchmarks/my-benchmark/tasks/remove \
+        -H 'Content-Type: application/json' \
+        -d '{
+          'taskSlugs': [
+            {'taskSlug': 'count-letters'},
+            {'ownerSlug': 'bob', 'taskSlug': 'sort-list', 'versionNumber': 2}
+          ]
+        }'
+
+    Args:
+      request (ApiRemoveTasksFromBenchmarkRequest):
+        The request object; initialized to empty instance if not specified.
+    """
+
+    if request is None:
+      request = ApiRemoveTasksFromBenchmarkRequest()
+
+    return self._client.call("benchmarks.BenchmarksApiService", "RemoveTasksFromBenchmark", request, ApiRemoveTasksFromBenchmarkResponse)
+
+  def register_auxiliary_model_version_configs(self, request: ApiRegisterAuxiliaryModelVersionConfigsRequest = None):
+    r"""
+    Retroactively records auxiliary BenchmarkModelVersionConfigs a task version
+    invokes beyond the candidate (e.g. LLM judges), for authors who did not
+    declare them at creation time. The task is resolved under the current user;
+    configs are addressed by slug. Additive and idempotent: configs already
+    registered on the version are skipped.
+
+    Args:
+      request (ApiRegisterAuxiliaryModelVersionConfigsRequest):
+        The request object; initialized to empty instance if not specified.
+    """
+
+    if request is None:
+      request = ApiRegisterAuxiliaryModelVersionConfigsRequest()
+
+    self._client.call("benchmarks.BenchmarksApiService", "RegisterAuxiliaryModelVersionConfigs", request, None)
+
+  def unregister_auxiliary_model_version_configs(self, request: ApiUnregisterAuxiliaryModelVersionConfigsRequest = None):
+    r"""
+    Removes previously-registered auxiliary BenchmarkModelVersionConfigs from a
+    task version. The task is resolved under the current user; configs are
+    addressed by slug. Idempotent: configs not currently registered are
+    ignored.
+
+    Args:
+      request (ApiUnregisterAuxiliaryModelVersionConfigsRequest):
+        The request object; initialized to empty instance if not specified.
+    """
+
+    if request is None:
+      request = ApiUnregisterAuxiliaryModelVersionConfigsRequest()
+
+    self._client.call("benchmarks.BenchmarksApiService", "UnregisterAuxiliaryModelVersionConfigs", request, None)
+
+  def get_benchmark_arena_settings(self, request: ApiGetBenchmarkArenaSettingsRequest = None) -> BenchmarkArenaSettings:
+    r"""
+    Get a benchmark version's arena settings. Requires update access to the
+    benchmark version. Returns NOT_FOUND if the version has no arena.
+
+    Example:
+      curl -sSL -u fickleone:local_api_token \
+        http://localhost/api/v1/benchmarks/fickleone/my-benchmark/arena/settings
+
+    Args:
+      request (ApiGetBenchmarkArenaSettingsRequest):
+        The request object; initialized to empty instance if not specified.
+    """
+
+    if request is None:
+      request = ApiGetBenchmarkArenaSettingsRequest()
+
+    return self._client.call("benchmarks.BenchmarksApiService", "GetBenchmarkArenaSettings", request, BenchmarkArenaSettings)
+
+  def create_benchmark_arena_settings(self, request: ApiCreateBenchmarkArenaSettingsRequest = None) -> BenchmarkArenaSettings:
+    r"""
+    Create a benchmark version's arena. Requires update access to the
+    benchmark version.
+
+    Example:
+      curl -sSL -u fickleone:local_api_token \
+        -X POST \
+        http://localhost/api/v1/benchmarks/fickleone/my-benchmark/arena/settings/create
+        \
+        -H 'Content-Type: application/json' \
+        -d '{
+          'settings': {
+            'artifactPath': '*.mp4',
+            'artifactType': 'BENCHMARK_ARENA_ARTIFACT_TYPE_VIDEO',
+            'revealCandidates': true
+          }
+        }'
+
+    Args:
+      request (ApiCreateBenchmarkArenaSettingsRequest):
+        The request object; initialized to empty instance if not specified.
+    """
+
+    if request is None:
+      request = ApiCreateBenchmarkArenaSettingsRequest()
+
+    return self._client.call("benchmarks.BenchmarksApiService", "CreateBenchmarkArenaSettings", request, BenchmarkArenaSettings)
+
+  def update_benchmark_arena_settings(self, request: ApiUpdateBenchmarkArenaSettingsRequest = None) -> BenchmarkArenaSettings:
+    r"""
+    Update a benchmark version's arena settings. Requires update access to the
+    benchmark version.
+
+    Example:
+      curl -sSL -u fickleone:local_api_token \
+        -X POST \
+        http://localhost/api/v1/benchmarks/fickleone/my-benchmark/arena/settings/update
+        \
+        -H 'Content-Type: application/json' \
+        -d '{
+          'settings': {'votingEnabled': true},
+          'updateMask': 'votingEnabled'
+        }'
+
+    Args:
+      request (ApiUpdateBenchmarkArenaSettingsRequest):
+        The request object; initialized to empty instance if not specified.
+    """
+
+    if request is None:
+      request = ApiUpdateBenchmarkArenaSettingsRequest()
+
+    return self._client.call("benchmarks.BenchmarksApiService", "UpdateBenchmarkArenaSettings", request, BenchmarkArenaSettings)
+
+  def list_benchmark_arena_combinations(self, request: ApiListBenchmarkArenaCombinationsRequest = None) -> ApiListBenchmarkArenaCombinationsResponse:
+    r"""
+    List the (task version, candidate) combinations opted into a benchmark
+    version's arena. Requires update access to the benchmark version.
+
+    Example:
+      curl -sSL -u fickleone:local_api_token \
+        http://localhost/api/v1/benchmarks/fickleone/my-benchmark/arena/combinations
+
+    Args:
+      request (ApiListBenchmarkArenaCombinationsRequest):
+        The request object; initialized to empty instance if not specified.
+    """
+
+    if request is None:
+      request = ApiListBenchmarkArenaCombinationsRequest()
+
+    return self._client.call("benchmarks.BenchmarksApiService", "ListBenchmarkArenaCombinations", request, ApiListBenchmarkArenaCombinationsResponse)
+
+  def update_benchmark_arena_combinations(self, request: ApiUpdateBenchmarkArenaCombinationsRequest = None):
+    r"""
+    Opt (task version, candidate) combinations into or out of a benchmark
+    version's arena. Requires update access to the benchmark version and read
+    access to each candidate added.
+
+    Example:
+      curl -sSL -u fickleone:local_api_token \
+        -X POST \
+        http://localhost/api/v1/benchmarks/fickleone/my-benchmark/arena/combinations/update
+        \
+        -H 'Content-Type: application/json' \
+        -d '{
+          'add': [
+            {'taskVersionId': 1, 'candidate': {'benchmarkModelVersionId': 2}}
+          ]
+        }'
+
+    Args:
+      request (ApiUpdateBenchmarkArenaCombinationsRequest):
+        The request object; initialized to empty instance if not specified.
+    """
+
+    if request is None:
+      request = ApiUpdateBenchmarkArenaCombinationsRequest()
+
+    self._client.call("benchmarks.BenchmarksApiService", "UpdateBenchmarkArenaCombinations", request, None)