Skip to content

Commit e5e40b8

Browse files
stainless-app[bot]stainless-app[bot]
committed
feat(api): Add tools and structured outputs to evals
1 parent 498fb99 commit e5e40b8

10 files changed

Lines changed: 314 additions & 2 deletions

‎.stats.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
configured_endpoints: 111
2-
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-0205acb1015d29b2312a48526734c0399f93026d4fe2dff5c7768f566e333fd2.yml
3-
openapi_spec_hash: 1772cc9056c2f6dfb2a4e9cb77ee6343
2+
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-4865dda2b62927bd141cbc85f81be3d88602f103e2c581e15eb1caded3e3aaa2.yml
3+
openapi_spec_hash: 7d14a9b23ef4ac93ea46d629601b6f6b
44
config_hash: ed1e6b3c5f93d12b80d31167f55c557c

‎src/openai/types/chat/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
from .chat_completion import ChatCompletion as ChatCompletion
66
from .chat_completion_role import ChatCompletionRole as ChatCompletionRole
7+
from .chat_completion_tool import ChatCompletionTool as ChatCompletionTool
78
from .chat_completion_audio import ChatCompletionAudio as ChatCompletionAudio
89
from .chat_completion_chunk import ChatCompletionChunk as ChatCompletionChunk
910
from .completion_list_params import CompletionListParams as CompletionListParams

‎src/openai/types/chat/chat_completion_tool.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2+
3+
from typing_extensions import Literal
4+
5+
from ..._models import BaseModel
6+
from ..shared.function_definition import FunctionDefinition
7+
8+
__all__ = ["ChatCompletionTool"]
9+
10+
11+
class ChatCompletionTool(BaseModel):
12+
function: FunctionDefinition
13+
14+
type: Literal["function"]
15+
"""The type of the tool. Currently, only `function` is supported."""

‎src/openai/types/evals/create_eval_completions_run_data_source.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,12 @@
66
from ..._utils import PropertyInfo
77
from ..._models import BaseModel
88
from ..shared.metadata import Metadata
9+
from ..chat.chat_completion_tool import ChatCompletionTool
10+
from ..shared.response_format_text import ResponseFormatText
911
from ..responses.easy_input_message import EasyInputMessage
1012
from ..responses.response_input_text import ResponseInputText
13+
from ..shared.response_format_json_object import ResponseFormatJSONObject
14+
from ..shared.response_format_json_schema import ResponseFormatJSONSchema
1115

1216
__all__ = [
1317
"CreateEvalCompletionsRunDataSource",
@@ -24,6 +28,7 @@
2428
"InputMessagesTemplateTemplateMessageContentOutputText",
2529
"InputMessagesItemReference",
2630
"SamplingParams",
31+
"SamplingParamsResponseFormat",
2732
]
2833

2934

@@ -136,17 +141,40 @@ class InputMessagesItemReference(BaseModel):
136141
Union[InputMessagesTemplate, InputMessagesItemReference], PropertyInfo(discriminator="type")
137142
]
138143

144+
SamplingParamsResponseFormat: TypeAlias = Union[ResponseFormatText, ResponseFormatJSONSchema, ResponseFormatJSONObject]
145+
139146

140147
class SamplingParams(BaseModel):
141148
max_completion_tokens: Optional[int] = None
142149
"""The maximum number of tokens in the generated output."""
143150

151+
response_format: Optional[SamplingParamsResponseFormat] = None
152+
"""An object specifying the format that the model must output.
153+
154+
Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured
155+
Outputs which ensures the model will match your supplied JSON schema. Learn more
156+
in the
157+
[Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
158+
159+
Setting to `{ "type": "json_object" }` enables the older JSON mode, which
160+
ensures the message the model generates is valid JSON. Using `json_schema` is
161+
preferred for models that support it.
162+
"""
163+
144164
seed: Optional[int] = None
145165
"""A seed value to initialize the randomness, during sampling."""
146166

147167
temperature: Optional[float] = None
148168
"""A higher temperature increases randomness in the outputs."""
149169

170+
tools: Optional[List[ChatCompletionTool]] = None
171+
"""A list of tools the model may call.
172+
173+
Currently, only functions are supported as a tool. Use this to provide a list of
174+
functions the model may generate JSON inputs for. A max of 128 functions are
175+
supported.
176+
"""
177+
150178
top_p: Optional[float] = None
151179
"""An alternative to temperature for nucleus sampling; 1.0 includes all tokens."""
152180

‎src/openai/types/evals/create_eval_completions_run_data_source_param.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,12 @@
66
from typing_extensions import Literal, Required, TypeAlias, TypedDict
77

88
from ..shared_params.metadata import Metadata
9+
from ..chat.chat_completion_tool_param import ChatCompletionToolParam
910
from ..responses.easy_input_message_param import EasyInputMessageParam
11+
from ..shared_params.response_format_text import ResponseFormatText
1012
from ..responses.response_input_text_param import ResponseInputTextParam
13+
from ..shared_params.response_format_json_object import ResponseFormatJSONObject
14+
from ..shared_params.response_format_json_schema import ResponseFormatJSONSchema
1115

1216
__all__ = [
1317
"CreateEvalCompletionsRunDataSourceParam",
@@ -24,6 +28,7 @@
2428
"InputMessagesTemplateTemplateMessageContentOutputText",
2529
"InputMessagesItemReference",
2630
"SamplingParams",
31+
"SamplingParamsResponseFormat",
2732
]
2833

2934

@@ -130,17 +135,40 @@ class InputMessagesItemReference(TypedDict, total=False):
130135

131136
InputMessages: TypeAlias = Union[InputMessagesTemplate, InputMessagesItemReference]
132137

138+
SamplingParamsResponseFormat: TypeAlias = Union[ResponseFormatText, ResponseFormatJSONSchema, ResponseFormatJSONObject]
139+
133140

134141
class SamplingParams(TypedDict, total=False):
135142
max_completion_tokens: int
136143
"""The maximum number of tokens in the generated output."""
137144

145+
response_format: SamplingParamsResponseFormat
146+
"""An object specifying the format that the model must output.
147+
148+
Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured
149+
Outputs which ensures the model will match your supplied JSON schema. Learn more
150+
in the
151+
[Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
152+
153+
Setting to `{ "type": "json_object" }` enables the older JSON mode, which
154+
ensures the message the model generates is valid JSON. Using `json_schema` is
155+
preferred for models that support it.
156+
"""
157+
138158
seed: int
139159
"""A seed value to initialize the randomness, during sampling."""
140160

141161
temperature: float
142162
"""A higher temperature increases randomness in the outputs."""
143163

164+
tools: Iterable[ChatCompletionToolParam]
165+
"""A list of tools the model may call.
166+
167+
Currently, only functions are supported as a tool. Use this to provide a list of
168+
functions the model may generate JSON inputs for. A max of 128 functions are
169+
supported.
170+
"""
171+
144172
top_p: float
145173
"""An alternative to temperature for nucleus sampling; 1.0 includes all tokens."""
146174

‎src/openai/types/evals/run_cancel_response.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,12 @@
88
from ..._utils import PropertyInfo
99
from ..._models import BaseModel
1010
from .eval_api_error import EvalAPIError
11+
from ..responses.tool import Tool
1112
from ..shared.metadata import Metadata
1213
from ..shared.reasoning_effort import ReasoningEffort
1314
from ..responses.response_input_text import ResponseInputText
1415
from .create_eval_jsonl_run_data_source import CreateEvalJSONLRunDataSource
16+
from ..responses.response_format_text_config import ResponseFormatTextConfig
1517
from .create_eval_completions_run_data_source import CreateEvalCompletionsRunDataSource
1618

1719
__all__ = [
@@ -32,6 +34,7 @@
3234
"DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText",
3335
"DataSourceResponsesInputMessagesItemReference",
3436
"DataSourceResponsesSamplingParams",
37+
"DataSourceResponsesSamplingParamsText",
3538
"PerModelUsage",
3639
"PerTestingCriteriaResult",
3740
"ResultCounts",
@@ -185,6 +188,24 @@ class DataSourceResponsesInputMessagesItemReference(BaseModel):
185188
]
186189

187190

191+
class DataSourceResponsesSamplingParamsText(BaseModel):
192+
format: Optional[ResponseFormatTextConfig] = None
193+
"""An object specifying the format that the model must output.
194+
195+
Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
196+
ensures the model will match your supplied JSON schema. Learn more in the
197+
[Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
198+
199+
The default format is `{ "type": "text" }` with no additional options.
200+
201+
**Not recommended for gpt-4o and newer models:**
202+
203+
Setting to `{ "type": "json_object" }` enables the older JSON mode, which
204+
ensures the message the model generates is valid JSON. Using `json_schema` is
205+
preferred for models that support it.
206+
"""
207+
208+
188209
class DataSourceResponsesSamplingParams(BaseModel):
189210
max_completion_tokens: Optional[int] = None
190211
"""The maximum number of tokens in the generated output."""
@@ -195,6 +216,33 @@ class DataSourceResponsesSamplingParams(BaseModel):
195216
temperature: Optional[float] = None
196217
"""A higher temperature increases randomness in the outputs."""
197218

219+
text: Optional[DataSourceResponsesSamplingParamsText] = None
220+
"""Configuration options for a text response from the model.
221+
222+
Can be plain text or structured JSON data. Learn more:
223+
224+
- [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
225+
- [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
226+
"""
227+
228+
tools: Optional[List[Tool]] = None
229+
"""An array of tools the model may call while generating a response.
230+
231+
You can specify which tool to use by setting the `tool_choice` parameter.
232+
233+
The two categories of tools you can provide the model are:
234+
235+
- **Built-in tools**: Tools that are provided by OpenAI that extend the model's
236+
capabilities, like
237+
[web search](https://platform.openai.com/docs/guides/tools-web-search) or
238+
[file search](https://platform.openai.com/docs/guides/tools-file-search).
239+
Learn more about
240+
[built-in tools](https://platform.openai.com/docs/guides/tools).
241+
- **Function calls (custom tools)**: Functions that are defined by you, enabling
242+
the model to call your own code. Learn more about
243+
[function calling](https://platform.openai.com/docs/guides/function-calling).
244+
"""
245+
198246
top_p: Optional[float] = None
199247
"""An alternative to temperature for nucleus sampling; 1.0 includes all tokens."""
200248

‎src/openai/types/evals/run_create_params.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55
from typing import Dict, List, Union, Iterable, Optional
66
from typing_extensions import Literal, Required, TypeAlias, TypedDict
77

8+
from ..responses.tool_param import ToolParam
89
from ..shared_params.metadata import Metadata
910
from ..shared.reasoning_effort import ReasoningEffort
1011
from ..responses.response_input_text_param import ResponseInputTextParam
1112
from .create_eval_jsonl_run_data_source_param import CreateEvalJSONLRunDataSourceParam
13+
from ..responses.response_format_text_config_param import ResponseFormatTextConfigParam
1214
from .create_eval_completions_run_data_source_param import CreateEvalCompletionsRunDataSourceParam
1315

1416
__all__ = [
@@ -29,6 +31,7 @@
2931
"DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateEvalItemContentOutputText",
3032
"DataSourceCreateEvalResponsesRunDataSourceInputMessagesItemReference",
3133
"DataSourceCreateEvalResponsesRunDataSourceSamplingParams",
34+
"DataSourceCreateEvalResponsesRunDataSourceSamplingParamsText",
3235
]
3336

3437

@@ -202,6 +205,24 @@ class DataSourceCreateEvalResponsesRunDataSourceInputMessagesItemReference(Typed
202205
]
203206

204207

208+
class DataSourceCreateEvalResponsesRunDataSourceSamplingParamsText(TypedDict, total=False):
209+
format: ResponseFormatTextConfigParam
210+
"""An object specifying the format that the model must output.
211+
212+
Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
213+
ensures the model will match your supplied JSON schema. Learn more in the
214+
[Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
215+
216+
The default format is `{ "type": "text" }` with no additional options.
217+
218+
**Not recommended for gpt-4o and newer models:**
219+
220+
Setting to `{ "type": "json_object" }` enables the older JSON mode, which
221+
ensures the message the model generates is valid JSON. Using `json_schema` is
222+
preferred for models that support it.
223+
"""
224+
225+
205226
class DataSourceCreateEvalResponsesRunDataSourceSamplingParams(TypedDict, total=False):
206227
max_completion_tokens: int
207228
"""The maximum number of tokens in the generated output."""
@@ -212,6 +233,33 @@ class DataSourceCreateEvalResponsesRunDataSourceSamplingParams(TypedDict, total=
212233
temperature: float
213234
"""A higher temperature increases randomness in the outputs."""
214235

236+
text: DataSourceCreateEvalResponsesRunDataSourceSamplingParamsText
237+
"""Configuration options for a text response from the model.
238+
239+
Can be plain text or structured JSON data. Learn more:
240+
241+
- [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
242+
- [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
243+
"""
244+
245+
tools: Iterable[ToolParam]
246+
"""An array of tools the model may call while generating a response.
247+
248+
You can specify which tool to use by setting the `tool_choice` parameter.
249+
250+
The two categories of tools you can provide the model are:
251+
252+
- **Built-in tools**: Tools that are provided by OpenAI that extend the model's
253+
capabilities, like
254+
[web search](https://platform.openai.com/docs/guides/tools-web-search) or
255+
[file search](https://platform.openai.com/docs/guides/tools-file-search).
256+
Learn more about
257+
[built-in tools](https://platform.openai.com/docs/guides/tools).
258+
- **Function calls (custom tools)**: Functions that are defined by you, enabling
259+
the model to call your own code. Learn more about
260+
[function calling](https://platform.openai.com/docs/guides/function-calling).
261+
"""
262+
215263
top_p: float
216264
"""An alternative to temperature for nucleus sampling; 1.0 includes all tokens."""
217265

‎src/openai/types/evals/run_create_response.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,12 @@
88
from ..._utils import PropertyInfo
99
from ..._models import BaseModel
1010
from .eval_api_error import EvalAPIError
11+
from ..responses.tool import Tool
1112
from ..shared.metadata import Metadata
1213
from ..shared.reasoning_effort import ReasoningEffort
1314
from ..responses.response_input_text import ResponseInputText
1415
from .create_eval_jsonl_run_data_source import CreateEvalJSONLRunDataSource
16+
from ..responses.response_format_text_config import ResponseFormatTextConfig
1517
from .create_eval_completions_run_data_source import CreateEvalCompletionsRunDataSource
1618

1719
__all__ = [
@@ -32,6 +34,7 @@
3234
"DataSourceResponsesInputMessagesTemplateTemplateEvalItemContentOutputText",
3335
"DataSourceResponsesInputMessagesItemReference",
3436
"DataSourceResponsesSamplingParams",
37+
"DataSourceResponsesSamplingParamsText",
3538
"PerModelUsage",
3639
"PerTestingCriteriaResult",
3740
"ResultCounts",
@@ -185,6 +188,24 @@ class DataSourceResponsesInputMessagesItemReference(BaseModel):
185188
]
186189

187190

191+
class DataSourceResponsesSamplingParamsText(BaseModel):
192+
format: Optional[ResponseFormatTextConfig] = None
193+
"""An object specifying the format that the model must output.
194+
195+
Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
196+
ensures the model will match your supplied JSON schema. Learn more in the
197+
[Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
198+
199+
The default format is `{ "type": "text" }` with no additional options.
200+
201+
**Not recommended for gpt-4o and newer models:**
202+
203+
Setting to `{ "type": "json_object" }` enables the older JSON mode, which
204+
ensures the message the model generates is valid JSON. Using `json_schema` is
205+
preferred for models that support it.
206+
"""
207+
208+
188209
class DataSourceResponsesSamplingParams(BaseModel):
189210
max_completion_tokens: Optional[int] = None
190211
"""The maximum number of tokens in the generated output."""
@@ -195,6 +216,33 @@ class DataSourceResponsesSamplingParams(BaseModel):
195216
temperature: Optional[float] = None
196217
"""A higher temperature increases randomness in the outputs."""
197218

219+
text: Optional[DataSourceResponsesSamplingParamsText] = None
220+
"""Configuration options for a text response from the model.
221+
222+
Can be plain text or structured JSON data. Learn more:
223+
224+
- [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
225+
- [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
226+
"""
227+
228+
tools: Optional[List[Tool]] = None
229+
"""An array of tools the model may call while generating a response.
230+
231+
You can specify which tool to use by setting the `tool_choice` parameter.
232+
233+
The two categories of tools you can provide the model are:
234+
235+
- **Built-in tools**: Tools that are provided by OpenAI that extend the model's
236+
capabilities, like
237+
[web search](https://platform.openai.com/docs/guides/tools-web-search) or
238+
[file search](https://platform.openai.com/docs/guides/tools-file-search).
239+
Learn more about
240+
[built-in tools](https://platform.openai.com/docs/guides/tools).
241+
- **Function calls (custom tools)**: Functions that are defined by you, enabling
242+
the model to call your own code. Learn more about
243+
[function calling](https://platform.openai.com/docs/guides/function-calling).
244+
"""
245+
198246
top_p: Optional[float] = None
199247
"""An alternative to temperature for nucleus sampling; 1.0 includes all tokens."""
200248

0 commit comments

Comments
 (0)