platform-admin/boc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Passwordless cross-device authentication

Browse Source 
- Arkitektur: docs/auth/passwordless-architecture.md
- Backend: iom/quixzoom-auth-service/ (FastAPI + Redis)
- Webb: quixzoom-market-pages/se/login/ (QR-kod + polling)
- App: iom/quixzoom-app/src/features/auth/ (push + deep links)

Flöde: QR-kod → app-godkännande → webb-inloggad
This commit is contained in:
Bernt
2026-07-07 07:11:50 +00:00
parent 4aa984ad74
commit 6989a98d75
61843 changed files with 5491611 additions and 872231 deletions
Download Patch File Download Diff File Expand all files Collapse all files
landvex/node_modules/openai/resources/evals/evals.d.mts
Generated Vendored
+739
View File
@@ -0,0 +1,739 @@
import { APIResource } from "../../core/resource.mjs";
import * as Shared from "../shared.mjs";
import * as GraderModelsAPI from "../graders/grader-models.mjs";
import * as ResponsesAPI from "../responses/responses.mjs";
import * as RunsAPI from "./runs/runs.mjs";
import { CreateEvalCompletionsRunDataSource, CreateEvalJSONLRunDataSource, EvalAPIError, RunCancelParams, RunCancelResponse, RunCreateParams, RunCreateResponse, RunDeleteParams, RunDeleteResponse, RunListParams, RunListResponse, RunListResponsesPage, RunRetrieveParams, RunRetrieveResponse, Runs } from "./runs/runs.mjs";
import { APIPromise } from "../../core/api-promise.mjs";
import { CursorPage, type CursorPageParams, PagePromise } from "../../core/pagination.mjs";
import { RequestOptions } from "../../internal/request-options.mjs";
/**
* Manage and run evals in the OpenAI platform.
*/
export declare class Evals extends APIResource {
runs: RunsAPI.Runs;
/**
* Create the structure of an evaluation that can be used to test a model's
* performance. An evaluation is a set of testing criteria and the config for a
* data source, which dictates the schema of the data used in the evaluation. After
* creating an evaluation, you can run it on different models and model parameters.
* We support several types of graders and datasources. For more information, see
* the [Evals guide](https://platform.openai.com/docs/guides/evals).
*/
create(body: EvalCreateParams, options?: RequestOptions): APIPromise<EvalCreateResponse>;
/**
* Get an evaluation by ID.
*/
retrieve(evalID: string, options?: RequestOptions): APIPromise<EvalRetrieveResponse>;
/**
* Update certain properties of an evaluation.
*/
update(evalID: string, body: EvalUpdateParams, options?: RequestOptions): APIPromise<EvalUpdateResponse>;
/**
* List evaluations for a project.
*/
list(query?: EvalListParams | null | undefined, options?: RequestOptions): PagePromise<EvalListResponsesPage, EvalListResponse>;
/**
* Delete an evaluation.
*/
delete(evalID: string, options?: RequestOptions): APIPromise<EvalDeleteResponse>;
}
export type EvalListResponsesPage = CursorPage<EvalListResponse>;
/**
* A CustomDataSourceConfig which specifies the schema of your `item` and
* optionally `sample` namespaces. The response schema defines the shape of the
* data that will be:
*
* - Used to define your testing criteria and
* - What data is required when creating a run
*/
export interface EvalCustomDataSourceConfig {
/**
* The json schema for the run data source items. Learn how to build JSON schemas
* [here](https://json-schema.org/).
*/
schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `custom`.
*/
type: 'custom';
}
/**
* @deprecated Deprecated in favor of LogsDataSourceConfig.
*/
export interface EvalStoredCompletionsDataSourceConfig {
/**
* The json schema for the run data source items. Learn how to build JSON schemas
* [here](https://json-schema.org/).
*/
schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `stored_completions`.
*/
type: 'stored_completions';
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
}
/**
* An Eval object with a data source config and testing criteria. An Eval
* represents a task to be done for your LLM integration. Like:
*
* - Improve the quality of my chatbot
* - See how well my chatbot handles customer support
* - Check if o4-mini is better at my usecase than gpt-4o
*/
export interface EvalCreateResponse {
/**
* Unique identifier for the evaluation.
*/
id: string;
/**
* The Unix timestamp (in seconds) for when the eval was created.
*/
created_at: number;
/**
* Configuration of data sources used in runs of the evaluation.
*/
data_source_config: EvalCustomDataSourceConfig | EvalCreateResponse.Logs | EvalStoredCompletionsDataSourceConfig;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The name of the evaluation.
*/
name: string;
/**
* The object type.
*/
object: 'eval';
/**
* A list of testing criteria.
*/
testing_criteria: Array<GraderModelsAPI.LabelModelGrader | GraderModelsAPI.StringCheckGrader | EvalCreateResponse.EvalGraderTextSimilarity | EvalCreateResponse.EvalGraderPython | EvalCreateResponse.EvalGraderScoreModel>;
}
export declare namespace EvalCreateResponse {
/**
* A LogsDataSourceConfig which specifies the metadata property of your logs query.
* This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. The
* schema returned by this data source config is used to defined what variables are
* available in your evals. `item` and `sample` are both defined when using this
* data source config.
*/
interface Logs {
/**
* The json schema for the run data source items. Learn how to build JSON schemas
* [here](https://json-schema.org/).
*/
schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `logs`.
*/
type: 'logs';
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
}
/**
* A TextSimilarityGrader object which grades text based on similarity metrics.
*/
interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
/**
* The threshold for the score.
*/
pass_threshold: number;
}
/**
* A PythonGrader object that runs a python script on the input.
*/
interface EvalGraderPython extends GraderModelsAPI.PythonGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
/**
* A ScoreModelGrader object that uses a model to assign a score to the input.
*/
interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
}
/**
* An Eval object with a data source config and testing criteria. An Eval
* represents a task to be done for your LLM integration. Like:
*
* - Improve the quality of my chatbot
* - See how well my chatbot handles customer support
* - Check if o4-mini is better at my usecase than gpt-4o
*/
export interface EvalRetrieveResponse {
/**
* Unique identifier for the evaluation.
*/
id: string;
/**
* The Unix timestamp (in seconds) for when the eval was created.
*/
created_at: number;
/**
* Configuration of data sources used in runs of the evaluation.
*/
data_source_config: EvalCustomDataSourceConfig | EvalRetrieveResponse.Logs | EvalStoredCompletionsDataSourceConfig;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The name of the evaluation.
*/
name: string;
/**
* The object type.
*/
object: 'eval';
/**
* A list of testing criteria.
*/
testing_criteria: Array<GraderModelsAPI.LabelModelGrader | GraderModelsAPI.StringCheckGrader | EvalRetrieveResponse.EvalGraderTextSimilarity | EvalRetrieveResponse.EvalGraderPython | EvalRetrieveResponse.EvalGraderScoreModel>;
}
export declare namespace EvalRetrieveResponse {
/**
* A LogsDataSourceConfig which specifies the metadata property of your logs query.
* This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. The
* schema returned by this data source config is used to defined what variables are
* available in your evals. `item` and `sample` are both defined when using this
* data source config.
*/
interface Logs {
/**
* The json schema for the run data source items. Learn how to build JSON schemas
* [here](https://json-schema.org/).
*/
schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `logs`.
*/
type: 'logs';
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
}
/**
* A TextSimilarityGrader object which grades text based on similarity metrics.
*/
interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
/**
* The threshold for the score.
*/
pass_threshold: number;
}
/**
* A PythonGrader object that runs a python script on the input.
*/
interface EvalGraderPython extends GraderModelsAPI.PythonGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
/**
* A ScoreModelGrader object that uses a model to assign a score to the input.
*/
interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
}
/**
* An Eval object with a data source config and testing criteria. An Eval
* represents a task to be done for your LLM integration. Like:
*
* - Improve the quality of my chatbot
* - See how well my chatbot handles customer support
* - Check if o4-mini is better at my usecase than gpt-4o
*/
export interface EvalUpdateResponse {
/**
* Unique identifier for the evaluation.
*/
id: string;
/**
* The Unix timestamp (in seconds) for when the eval was created.
*/
created_at: number;
/**
* Configuration of data sources used in runs of the evaluation.
*/
data_source_config: EvalCustomDataSourceConfig | EvalUpdateResponse.Logs | EvalStoredCompletionsDataSourceConfig;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The name of the evaluation.
*/
name: string;
/**
* The object type.
*/
object: 'eval';
/**
* A list of testing criteria.
*/
testing_criteria: Array<GraderModelsAPI.LabelModelGrader | GraderModelsAPI.StringCheckGrader | EvalUpdateResponse.EvalGraderTextSimilarity | EvalUpdateResponse.EvalGraderPython | EvalUpdateResponse.EvalGraderScoreModel>;
}
export declare namespace EvalUpdateResponse {
/**
* A LogsDataSourceConfig which specifies the metadata property of your logs query.
* This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. The
* schema returned by this data source config is used to defined what variables are
* available in your evals. `item` and `sample` are both defined when using this
* data source config.
*/
interface Logs {
/**
* The json schema for the run data source items. Learn how to build JSON schemas
* [here](https://json-schema.org/).
*/
schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `logs`.
*/
type: 'logs';
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
}
/**
* A TextSimilarityGrader object which grades text based on similarity metrics.
*/
interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
/**
* The threshold for the score.
*/
pass_threshold: number;
}
/**
* A PythonGrader object that runs a python script on the input.
*/
interface EvalGraderPython extends GraderModelsAPI.PythonGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
/**
* A ScoreModelGrader object that uses a model to assign a score to the input.
*/
interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
}
/**
* An Eval object with a data source config and testing criteria. An Eval
* represents a task to be done for your LLM integration. Like:
*
* - Improve the quality of my chatbot
* - See how well my chatbot handles customer support
* - Check if o4-mini is better at my usecase than gpt-4o
*/
export interface EvalListResponse {
/**
* Unique identifier for the evaluation.
*/
id: string;
/**
* The Unix timestamp (in seconds) for when the eval was created.
*/
created_at: number;
/**
* Configuration of data sources used in runs of the evaluation.
*/
data_source_config: EvalCustomDataSourceConfig | EvalListResponse.Logs | EvalStoredCompletionsDataSourceConfig;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The name of the evaluation.
*/
name: string;
/**
* The object type.
*/
object: 'eval';
/**
* A list of testing criteria.
*/
testing_criteria: Array<GraderModelsAPI.LabelModelGrader | GraderModelsAPI.StringCheckGrader | EvalListResponse.EvalGraderTextSimilarity | EvalListResponse.EvalGraderPython | EvalListResponse.EvalGraderScoreModel>;
}
export declare namespace EvalListResponse {
/**
* A LogsDataSourceConfig which specifies the metadata property of your logs query.
* This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. The
* schema returned by this data source config is used to defined what variables are
* available in your evals. `item` and `sample` are both defined when using this
* data source config.
*/
interface Logs {
/**
* The json schema for the run data source items. Learn how to build JSON schemas
* [here](https://json-schema.org/).
*/
schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `logs`.
*/
type: 'logs';
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
}
/**
* A TextSimilarityGrader object which grades text based on similarity metrics.
*/
interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
/**
* The threshold for the score.
*/
pass_threshold: number;
}
/**
* A PythonGrader object that runs a python script on the input.
*/
interface EvalGraderPython extends GraderModelsAPI.PythonGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
/**
* A ScoreModelGrader object that uses a model to assign a score to the input.
*/
interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
}
export interface EvalDeleteResponse {
deleted: boolean;
eval_id: string;
object: string;
}
export interface EvalCreateParams {
/**
* The configuration for the data source used for the evaluation runs. Dictates the
* schema of the data used in the evaluation.
*/
data_source_config: EvalCreateParams.Custom | EvalCreateParams.Logs | EvalCreateParams.StoredCompletions;
/**
* A list of graders for all eval runs in this group. Graders can reference
* variables in the data source using double curly braces notation, like
* `{{item.variable_name}}`. To reference the model's output, use the `sample`
* namespace (ie, `{{sample.output_text}}`).
*/
testing_criteria: Array<EvalCreateParams.LabelModel | GraderModelsAPI.StringCheckGrader | EvalCreateParams.TextSimilarity | EvalCreateParams.Python | EvalCreateParams.ScoreModel>;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
/**
* The name of the evaluation.
*/
name?: string;
}
export declare namespace EvalCreateParams {
/**
* A CustomDataSourceConfig object that defines the schema for the data source used
* for the evaluation runs. This schema is used to define the shape of the data
* that will be:
*
* - Used to define your testing criteria and
* - What data is required when creating a run
*/
interface Custom {
/**
* The json schema for each row in the data source.
*/
item_schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `custom`.
*/
type: 'custom';
/**
* Whether the eval should expect you to populate the sample namespace (ie, by
* generating responses off of your data source)
*/
include_sample_schema?: boolean;
}
/**
* A data source config which specifies the metadata property of your logs query.
* This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc.
*/
interface Logs {
/**
* The type of data source. Always `logs`.
*/
type: 'logs';
/**
* Metadata filters for the logs data source.
*/
metadata?: {
[key: string]: unknown;
};
}
/**
* @deprecated Deprecated in favor of LogsDataSourceConfig.
*/
interface StoredCompletions {
/**
* The type of data source. Always `stored_completions`.
*/
type: 'stored_completions';
/**
* Metadata filters for the stored completions data source.
*/
metadata?: {
[key: string]: unknown;
};
}
/**
* A LabelModelGrader object which uses a model to assign labels to each item in
* the evaluation.
*/
interface LabelModel {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
input: Array<LabelModel.SimpleInputMessage | LabelModel.EvalItem>;
/**
* The labels to classify to each item in the evaluation.
*/
labels: Array<string>;
/**
* The model to use for the evaluation. Must support structured outputs.
*/
model: string;
/**
* The name of the grader.
*/
name: string;
/**
* The labels that indicate a passing result. Must be a subset of labels.
*/
passing_labels: Array<string>;
/**
* The object type, which is always `label_model`.
*/
type: 'label_model';
}
namespace LabelModel {
interface SimpleInputMessage {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role: string;
}
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
/**
* A TextSimilarityGrader object which grades text based on similarity metrics.
*/
interface TextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
/**
* The threshold for the score.
*/
pass_threshold: number;
}
/**
* A PythonGrader object that runs a python script on the input.
*/
interface Python extends GraderModelsAPI.PythonGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
/**
* A ScoreModelGrader object that uses a model to assign a score to the input.
*/
interface ScoreModel extends GraderModelsAPI.ScoreModelGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
}
export interface EvalUpdateParams {
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
/**
* Rename the evaluation.
*/
name?: string;
}
export interface EvalListParams extends CursorPageParams {
/**
* Sort order for evals by timestamp. Use `asc` for ascending order or `desc` for
* descending order.
*/
order?: 'asc' | 'desc';
/**
* Evals can be ordered by creation time or last updated time. Use `created_at` for
* creation time or `updated_at` for last updated time.
*/
order_by?: 'created_at' | 'updated_at';
}
export declare namespace Evals {
export { type EvalCustomDataSourceConfig as EvalCustomDataSourceConfig, type EvalStoredCompletionsDataSourceConfig as EvalStoredCompletionsDataSourceConfig, type EvalCreateResponse as EvalCreateResponse, type EvalRetrieveResponse as EvalRetrieveResponse, type EvalUpdateResponse as EvalUpdateResponse, type EvalListResponse as EvalListResponse, type EvalDeleteResponse as EvalDeleteResponse, type EvalListResponsesPage as EvalListResponsesPage, type EvalCreateParams as EvalCreateParams, type EvalUpdateParams as EvalUpdateParams, type EvalListParams as EvalListParams, };
export { Runs as Runs, type CreateEvalCompletionsRunDataSource as CreateEvalCompletionsRunDataSource, type CreateEvalJSONLRunDataSource as CreateEvalJSONLRunDataSource, type EvalAPIError as EvalAPIError, type RunCreateResponse as RunCreateResponse, type RunRetrieveResponse as RunRetrieveResponse, type RunListResponse as RunListResponse, type RunDeleteResponse as RunDeleteResponse, type RunCancelResponse as RunCancelResponse, type RunListResponsesPage as RunListResponsesPage, type RunCreateParams as RunCreateParams, type RunRetrieveParams as RunRetrieveParams, type RunListParams as RunListParams, type RunDeleteParams as RunDeleteParams, type RunCancelParams as RunCancelParams, };
}
//# sourceMappingURL=evals.d.mts.map
landvex/node_modules/openai/resources/evals/evals.d.mts.map
Generated Vendored
+1
View File
File diff suppressed because one or more lines are too long
landvex/node_modules/openai/resources/evals/evals.d.ts
Generated Vendored
+739
View File
@@ -0,0 +1,739 @@
import { APIResource } from "../../core/resource.js";
import * as Shared from "../shared.js";
import * as GraderModelsAPI from "../graders/grader-models.js";
import * as ResponsesAPI from "../responses/responses.js";
import * as RunsAPI from "./runs/runs.js";
import { CreateEvalCompletionsRunDataSource, CreateEvalJSONLRunDataSource, EvalAPIError, RunCancelParams, RunCancelResponse, RunCreateParams, RunCreateResponse, RunDeleteParams, RunDeleteResponse, RunListParams, RunListResponse, RunListResponsesPage, RunRetrieveParams, RunRetrieveResponse, Runs } from "./runs/runs.js";
import { APIPromise } from "../../core/api-promise.js";
import { CursorPage, type CursorPageParams, PagePromise } from "../../core/pagination.js";
import { RequestOptions } from "../../internal/request-options.js";
/**
* Manage and run evals in the OpenAI platform.
*/
export declare class Evals extends APIResource {
runs: RunsAPI.Runs;
/**
* Create the structure of an evaluation that can be used to test a model's
* performance. An evaluation is a set of testing criteria and the config for a
* data source, which dictates the schema of the data used in the evaluation. After
* creating an evaluation, you can run it on different models and model parameters.
* We support several types of graders and datasources. For more information, see
* the [Evals guide](https://platform.openai.com/docs/guides/evals).
*/
create(body: EvalCreateParams, options?: RequestOptions): APIPromise<EvalCreateResponse>;
/**
* Get an evaluation by ID.
*/
retrieve(evalID: string, options?: RequestOptions): APIPromise<EvalRetrieveResponse>;
/**
* Update certain properties of an evaluation.
*/
update(evalID: string, body: EvalUpdateParams, options?: RequestOptions): APIPromise<EvalUpdateResponse>;
/**
* List evaluations for a project.
*/
list(query?: EvalListParams | null | undefined, options?: RequestOptions): PagePromise<EvalListResponsesPage, EvalListResponse>;
/**
* Delete an evaluation.
*/
delete(evalID: string, options?: RequestOptions): APIPromise<EvalDeleteResponse>;
}
export type EvalListResponsesPage = CursorPage<EvalListResponse>;
/**
* A CustomDataSourceConfig which specifies the schema of your `item` and
* optionally `sample` namespaces. The response schema defines the shape of the
* data that will be:
*
* - Used to define your testing criteria and
* - What data is required when creating a run
*/
export interface EvalCustomDataSourceConfig {
/**
* The json schema for the run data source items. Learn how to build JSON schemas
* [here](https://json-schema.org/).
*/
schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `custom`.
*/
type: 'custom';
}
/**
* @deprecated Deprecated in favor of LogsDataSourceConfig.
*/
export interface EvalStoredCompletionsDataSourceConfig {
/**
* The json schema for the run data source items. Learn how to build JSON schemas
* [here](https://json-schema.org/).
*/
schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `stored_completions`.
*/
type: 'stored_completions';
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
}
/**
* An Eval object with a data source config and testing criteria. An Eval
* represents a task to be done for your LLM integration. Like:
*
* - Improve the quality of my chatbot
* - See how well my chatbot handles customer support
* - Check if o4-mini is better at my usecase than gpt-4o
*/
export interface EvalCreateResponse {
/**
* Unique identifier for the evaluation.
*/
id: string;
/**
* The Unix timestamp (in seconds) for when the eval was created.
*/
created_at: number;
/**
* Configuration of data sources used in runs of the evaluation.
*/
data_source_config: EvalCustomDataSourceConfig | EvalCreateResponse.Logs | EvalStoredCompletionsDataSourceConfig;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The name of the evaluation.
*/
name: string;
/**
* The object type.
*/
object: 'eval';
/**
* A list of testing criteria.
*/
testing_criteria: Array<GraderModelsAPI.LabelModelGrader | GraderModelsAPI.StringCheckGrader | EvalCreateResponse.EvalGraderTextSimilarity | EvalCreateResponse.EvalGraderPython | EvalCreateResponse.EvalGraderScoreModel>;
}
export declare namespace EvalCreateResponse {
/**
* A LogsDataSourceConfig which specifies the metadata property of your logs query.
* This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. The
* schema returned by this data source config is used to defined what variables are
* available in your evals. `item` and `sample` are both defined when using this
* data source config.
*/
interface Logs {
/**
* The json schema for the run data source items. Learn how to build JSON schemas
* [here](https://json-schema.org/).
*/
schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `logs`.
*/
type: 'logs';
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
}
/**
* A TextSimilarityGrader object which grades text based on similarity metrics.
*/
interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
/**
* The threshold for the score.
*/
pass_threshold: number;
}
/**
* A PythonGrader object that runs a python script on the input.
*/
interface EvalGraderPython extends GraderModelsAPI.PythonGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
/**
* A ScoreModelGrader object that uses a model to assign a score to the input.
*/
interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
}
/**
* An Eval object with a data source config and testing criteria. An Eval
* represents a task to be done for your LLM integration. Like:
*
* - Improve the quality of my chatbot
* - See how well my chatbot handles customer support
* - Check if o4-mini is better at my usecase than gpt-4o
*/
export interface EvalRetrieveResponse {
/**
* Unique identifier for the evaluation.
*/
id: string;
/**
* The Unix timestamp (in seconds) for when the eval was created.
*/
created_at: number;
/**
* Configuration of data sources used in runs of the evaluation.
*/
data_source_config: EvalCustomDataSourceConfig | EvalRetrieveResponse.Logs | EvalStoredCompletionsDataSourceConfig;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The name of the evaluation.
*/
name: string;
/**
* The object type.
*/
object: 'eval';
/**
* A list of testing criteria.
*/
testing_criteria: Array<GraderModelsAPI.LabelModelGrader | GraderModelsAPI.StringCheckGrader | EvalRetrieveResponse.EvalGraderTextSimilarity | EvalRetrieveResponse.EvalGraderPython | EvalRetrieveResponse.EvalGraderScoreModel>;
}
export declare namespace EvalRetrieveResponse {
/**
* A LogsDataSourceConfig which specifies the metadata property of your logs query.
* This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. The
* schema returned by this data source config is used to defined what variables are
* available in your evals. `item` and `sample` are both defined when using this
* data source config.
*/
interface Logs {
/**
* The json schema for the run data source items. Learn how to build JSON schemas
* [here](https://json-schema.org/).
*/
schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `logs`.
*/
type: 'logs';
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
}
/**
* A TextSimilarityGrader object which grades text based on similarity metrics.
*/
interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
/**
* The threshold for the score.
*/
pass_threshold: number;
}
/**
* A PythonGrader object that runs a python script on the input.
*/
interface EvalGraderPython extends GraderModelsAPI.PythonGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
/**
* A ScoreModelGrader object that uses a model to assign a score to the input.
*/
interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
}
/**
* An Eval object with a data source config and testing criteria. An Eval
* represents a task to be done for your LLM integration. Like:
*
* - Improve the quality of my chatbot
* - See how well my chatbot handles customer support
* - Check if o4-mini is better at my usecase than gpt-4o
*/
export interface EvalUpdateResponse {
/**
* Unique identifier for the evaluation.
*/
id: string;
/**
* The Unix timestamp (in seconds) for when the eval was created.
*/
created_at: number;
/**
* Configuration of data sources used in runs of the evaluation.
*/
data_source_config: EvalCustomDataSourceConfig | EvalUpdateResponse.Logs | EvalStoredCompletionsDataSourceConfig;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The name of the evaluation.
*/
name: string;
/**
* The object type.
*/
object: 'eval';
/**
* A list of testing criteria.
*/
testing_criteria: Array<GraderModelsAPI.LabelModelGrader | GraderModelsAPI.StringCheckGrader | EvalUpdateResponse.EvalGraderTextSimilarity | EvalUpdateResponse.EvalGraderPython | EvalUpdateResponse.EvalGraderScoreModel>;
}
export declare namespace EvalUpdateResponse {
/**
* A LogsDataSourceConfig which specifies the metadata property of your logs query.
* This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. The
* schema returned by this data source config is used to defined what variables are
* available in your evals. `item` and `sample` are both defined when using this
* data source config.
*/
interface Logs {
/**
* The json schema for the run data source items. Learn how to build JSON schemas
* [here](https://json-schema.org/).
*/
schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `logs`.
*/
type: 'logs';
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
}
/**
* A TextSimilarityGrader object which grades text based on similarity metrics.
*/
interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
/**
* The threshold for the score.
*/
pass_threshold: number;
}
/**
* A PythonGrader object that runs a python script on the input.
*/
interface EvalGraderPython extends GraderModelsAPI.PythonGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
/**
* A ScoreModelGrader object that uses a model to assign a score to the input.
*/
interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
}
/**
* An Eval object with a data source config and testing criteria. An Eval
* represents a task to be done for your LLM integration. Like:
*
* - Improve the quality of my chatbot
* - See how well my chatbot handles customer support
* - Check if o4-mini is better at my usecase than gpt-4o
*/
export interface EvalListResponse {
/**
* Unique identifier for the evaluation.
*/
id: string;
/**
* The Unix timestamp (in seconds) for when the eval was created.
*/
created_at: number;
/**
* Configuration of data sources used in runs of the evaluation.
*/
data_source_config: EvalCustomDataSourceConfig | EvalListResponse.Logs | EvalStoredCompletionsDataSourceConfig;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The name of the evaluation.
*/
name: string;
/**
* The object type.
*/
object: 'eval';
/**
* A list of testing criteria.
*/
testing_criteria: Array<GraderModelsAPI.LabelModelGrader | GraderModelsAPI.StringCheckGrader | EvalListResponse.EvalGraderTextSimilarity | EvalListResponse.EvalGraderPython | EvalListResponse.EvalGraderScoreModel>;
}
export declare namespace EvalListResponse {
/**
* A LogsDataSourceConfig which specifies the metadata property of your logs query.
* This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc. The
* schema returned by this data source config is used to defined what variables are
* available in your evals. `item` and `sample` are both defined when using this
* data source config.
*/
interface Logs {
/**
* The json schema for the run data source items. Learn how to build JSON schemas
* [here](https://json-schema.org/).
*/
schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `logs`.
*/
type: 'logs';
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
}
/**
* A TextSimilarityGrader object which grades text based on similarity metrics.
*/
interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
/**
* The threshold for the score.
*/
pass_threshold: number;
}
/**
* A PythonGrader object that runs a python script on the input.
*/
interface EvalGraderPython extends GraderModelsAPI.PythonGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
/**
* A ScoreModelGrader object that uses a model to assign a score to the input.
*/
interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
}
export interface EvalDeleteResponse {
deleted: boolean;
eval_id: string;
object: string;
}
export interface EvalCreateParams {
/**
* The configuration for the data source used for the evaluation runs. Dictates the
* schema of the data used in the evaluation.
*/
data_source_config: EvalCreateParams.Custom | EvalCreateParams.Logs | EvalCreateParams.StoredCompletions;
/**
* A list of graders for all eval runs in this group. Graders can reference
* variables in the data source using double curly braces notation, like
* `{{item.variable_name}}`. To reference the model's output, use the `sample`
* namespace (ie, `{{sample.output_text}}`).
*/
testing_criteria: Array<EvalCreateParams.LabelModel | GraderModelsAPI.StringCheckGrader | EvalCreateParams.TextSimilarity | EvalCreateParams.Python | EvalCreateParams.ScoreModel>;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
/**
* The name of the evaluation.
*/
name?: string;
}
export declare namespace EvalCreateParams {
/**
* A CustomDataSourceConfig object that defines the schema for the data source used
* for the evaluation runs. This schema is used to define the shape of the data
* that will be:
*
* - Used to define your testing criteria and
* - What data is required when creating a run
*/
interface Custom {
/**
* The json schema for each row in the data source.
*/
item_schema: {
[key: string]: unknown;
};
/**
* The type of data source. Always `custom`.
*/
type: 'custom';
/**
* Whether the eval should expect you to populate the sample namespace (ie, by
* generating responses off of your data source)
*/
include_sample_schema?: boolean;
}
/**
* A data source config which specifies the metadata property of your logs query.
* This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc.
*/
interface Logs {
/**
* The type of data source. Always `logs`.
*/
type: 'logs';
/**
* Metadata filters for the logs data source.
*/
metadata?: {
[key: string]: unknown;
};
}
/**
* @deprecated Deprecated in favor of LogsDataSourceConfig.
*/
interface StoredCompletions {
/**
* The type of data source. Always `stored_completions`.
*/
type: 'stored_completions';
/**
* Metadata filters for the stored completions data source.
*/
metadata?: {
[key: string]: unknown;
};
}
/**
* A LabelModelGrader object which uses a model to assign labels to each item in
* the evaluation.
*/
interface LabelModel {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
input: Array<LabelModel.SimpleInputMessage | LabelModel.EvalItem>;
/**
* The labels to classify to each item in the evaluation.
*/
labels: Array<string>;
/**
* The model to use for the evaluation. Must support structured outputs.
*/
model: string;
/**
* The name of the grader.
*/
name: string;
/**
* The labels that indicate a passing result. Must be a subset of labels.
*/
passing_labels: Array<string>;
/**
* The object type, which is always `label_model`.
*/
type: 'label_model';
}
namespace LabelModel {
interface SimpleInputMessage {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role: string;
}
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
/**
* A TextSimilarityGrader object which grades text based on similarity metrics.
*/
interface TextSimilarity extends GraderModelsAPI.TextSimilarityGrader {
/**
* The threshold for the score.
*/
pass_threshold: number;
}
/**
* A PythonGrader object that runs a python script on the input.
*/
interface Python extends GraderModelsAPI.PythonGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
/**
* A ScoreModelGrader object that uses a model to assign a score to the input.
*/
interface ScoreModel extends GraderModelsAPI.ScoreModelGrader {
/**
* The threshold for the score.
*/
pass_threshold?: number;
}
}
export interface EvalUpdateParams {
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
/**
* Rename the evaluation.
*/
name?: string;
}
export interface EvalListParams extends CursorPageParams {
/**
* Sort order for evals by timestamp. Use `asc` for ascending order or `desc` for
* descending order.
*/
order?: 'asc' | 'desc';
/**
* Evals can be ordered by creation time or last updated time. Use `created_at` for
* creation time or `updated_at` for last updated time.
*/
order_by?: 'created_at' | 'updated_at';
}
export declare namespace Evals {
export { type EvalCustomDataSourceConfig as EvalCustomDataSourceConfig, type EvalStoredCompletionsDataSourceConfig as EvalStoredCompletionsDataSourceConfig, type EvalCreateResponse as EvalCreateResponse, type EvalRetrieveResponse as EvalRetrieveResponse, type EvalUpdateResponse as EvalUpdateResponse, type EvalListResponse as EvalListResponse, type EvalDeleteResponse as EvalDeleteResponse, type EvalListResponsesPage as EvalListResponsesPage, type EvalCreateParams as EvalCreateParams, type EvalUpdateParams as EvalUpdateParams, type EvalListParams as EvalListParams, };
export { Runs as Runs, type CreateEvalCompletionsRunDataSource as CreateEvalCompletionsRunDataSource, type CreateEvalJSONLRunDataSource as CreateEvalJSONLRunDataSource, type EvalAPIError as EvalAPIError, type RunCreateResponse as RunCreateResponse, type RunRetrieveResponse as RunRetrieveResponse, type RunListResponse as RunListResponse, type RunDeleteResponse as RunDeleteResponse, type RunCancelResponse as RunCancelResponse, type RunListResponsesPage as RunListResponsesPage, type RunCreateParams as RunCreateParams, type RunRetrieveParams as RunRetrieveParams, type RunListParams as RunListParams, type RunDeleteParams as RunDeleteParams, type RunCancelParams as RunCancelParams, };
}
//# sourceMappingURL=evals.d.ts.map
landvex/node_modules/openai/resources/evals/evals.d.ts.map
Generated Vendored
+1
View File
File diff suppressed because one or more lines are too long
landvex/node_modules/openai/resources/evals/evals.js
Generated Vendored
+61
View File
@@ -0,0 +1,61 @@
"use strict";
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
Object.defineProperty(exports, "__esModule", { value: true });
exports.Evals = void 0;
const tslib_1 = require("../../internal/tslib.js");
const resource_1 = require("../../core/resource.js");
const RunsAPI = tslib_1.__importStar(require("./runs/runs.js"));
const runs_1 = require("./runs/runs.js");
const pagination_1 = require("../../core/pagination.js");
const path_1 = require("../../internal/utils/path.js");
/**
* Manage and run evals in the OpenAI platform.
*/
class Evals extends resource_1.APIResource {
constructor() {
super(...arguments);
this.runs = new RunsAPI.Runs(this._client);
}
/**
* Create the structure of an evaluation that can be used to test a model's
* performance. An evaluation is a set of testing criteria and the config for a
* data source, which dictates the schema of the data used in the evaluation. After
* creating an evaluation, you can run it on different models and model parameters.
* We support several types of graders and datasources. For more information, see
* the [Evals guide](https://platform.openai.com/docs/guides/evals).
*/
create(body, options) {
return this._client.post('/evals', { body, ...options, __security: { bearerAuth: true } });
}
/**
* Get an evaluation by ID.
*/
retrieve(evalID, options) {
return this._client.get((0, path_1.path) `/evals/${evalID}`, { ...options, __security: { bearerAuth: true } });
}
/**
* Update certain properties of an evaluation.
*/
update(evalID, body, options) {
return this._client.post((0, path_1.path) `/evals/${evalID}`, { body, ...options, __security: { bearerAuth: true } });
}
/**
* List evaluations for a project.
*/
list(query = {}, options) {
return this._client.getAPIList('/evals', (pagination_1.CursorPage), {
query,
...options,
__security: { bearerAuth: true },
});
}
/**
* Delete an evaluation.
*/
delete(evalID, options) {
return this._client.delete((0, path_1.path) `/evals/${evalID}`, { ...options, __security: { bearerAuth: true } });
}
}
exports.Evals = Evals;
Evals.Runs = runs_1.Runs;
//# sourceMappingURL=evals.js.map
landvex/node_modules/openai/resources/evals/evals.js.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"evals.js","sourceRoot":"","sources":["../../src/resources/evals/evals.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;;AAEtF,qDAAkD;AAIlD,gEAAuC;AACvC,yCAgBqB;AAErB,yDAAuF;AAEvF,uDAAiD;AAEjD;;GAEG;AACH,MAAa,KAAM,SAAQ,sBAAW;IAAtC;;QACE,SAAI,GAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAgDtD,CAAC;IA9CC;;;;;;;OAOG;IACH,MAAM,CAAC,IAAsB,EAAE,OAAwB;QACrD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IAC7F,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,MAAc,EAAE,OAAwB;QAC/C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,UAAU,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IACpG,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,MAAc,EAAE,IAAsB,EAAE,OAAwB;QACrE,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAA,WAAI,EAAA,UAAU,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IAC3G,CAAC;IAED;;OAEG;IACH,IAAI,CACF,QAA2C,EAAE,EAC7C,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAA,uBAA4B,CAAA,EAAE;YACrE,KAAK;YACL,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,MAAc,EAAE,OAAwB;QAC7C,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAA,WAAI,EAAA,UAAU,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IACvG,CAAC;CACF;AAjDD,sBAiDC;AAqzBD,KAAK,CAAC,IAAI,GAAG,WAAI,CAAC"}
landvex/node_modules/openai/resources/evals/evals.mjs
Generated Vendored
+56
View File
@@ -0,0 +1,56 @@
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
import { APIResource } from "../../core/resource.mjs";
import * as RunsAPI from "./runs/runs.mjs";
import { Runs, } from "./runs/runs.mjs";
import { CursorPage } from "../../core/pagination.mjs";
import { path } from "../../internal/utils/path.mjs";
/**
* Manage and run evals in the OpenAI platform.
*/
export class Evals extends APIResource {
constructor() {
super(...arguments);
this.runs = new RunsAPI.Runs(this._client);
}
/**
* Create the structure of an evaluation that can be used to test a model's
* performance. An evaluation is a set of testing criteria and the config for a
* data source, which dictates the schema of the data used in the evaluation. After
* creating an evaluation, you can run it on different models and model parameters.
* We support several types of graders and datasources. For more information, see
* the [Evals guide](https://platform.openai.com/docs/guides/evals).
*/
create(body, options) {
return this._client.post('/evals', { body, ...options, __security: { bearerAuth: true } });
}
/**
* Get an evaluation by ID.
*/
retrieve(evalID, options) {
return this._client.get(path `/evals/${evalID}`, { ...options, __security: { bearerAuth: true } });
}
/**
* Update certain properties of an evaluation.
*/
update(evalID, body, options) {
return this._client.post(path `/evals/${evalID}`, { body, ...options, __security: { bearerAuth: true } });
}
/**
* List evaluations for a project.
*/
list(query = {}, options) {
return this._client.getAPIList('/evals', (CursorPage), {
query,
...options,
__security: { bearerAuth: true },
});
}
/**
* Delete an evaluation.
*/
delete(evalID, options) {
return this._client.delete(path `/evals/${evalID}`, { ...options, __security: { bearerAuth: true } });
}
}
Evals.Runs = Runs;
//# sourceMappingURL=evals.mjs.map
landvex/node_modules/openai/resources/evals/evals.mjs.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"evals.mjs","sourceRoot":"","sources":["../../src/resources/evals/evals.ts"],"names":[],"mappings":"AAAA,sFAAsF;AAEtF,OAAO,EAAE,WAAW,EAAE,gCAA4B;AAIlD,OAAO,KAAK,OAAO,wBAAoB;AACvC,OAAO,EAeL,IAAI,GACL,wBAAoB;AAErB,OAAO,EAAE,UAAU,EAAsC,kCAA8B;AAEvF,OAAO,EAAE,IAAI,EAAE,sCAAkC;AAEjD;;GAEG;AACH,MAAM,OAAO,KAAM,SAAQ,WAAW;IAAtC;;QACE,SAAI,GAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAgDtD,CAAC;IA9CC;;;;;;;OAOG;IACH,MAAM,CAAC,IAAsB,EAAE,OAAwB;QACrD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IAC7F,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,MAAc,EAAE,OAAwB;QAC/C,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,UAAU,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IACpG,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,MAAc,EAAE,IAAsB,EAAE,OAAwB;QACrE,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAA,UAAU,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IAC3G,CAAC;IAED;;OAEG;IACH,IAAI,CACF,QAA2C,EAAE,EAC7C,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAA,UAA4B,CAAA,EAAE;YACrE,KAAK;YACL,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,MAAc,EAAE,OAAwB;QAC7C,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAA,UAAU,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IACvG,CAAC;CACF;AAqzBD,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC"}
landvex/node_modules/openai/resources/evals/index.d.mts
Generated Vendored
+3
View File
@@ -0,0 +1,3 @@
export { Evals, type EvalCustomDataSourceConfig, type EvalStoredCompletionsDataSourceConfig, type EvalCreateResponse, type EvalRetrieveResponse, type EvalUpdateResponse, type EvalListResponse, type EvalDeleteResponse, type EvalCreateParams, type EvalUpdateParams, type EvalListParams, type EvalListResponsesPage, } from "./evals.mjs";
export { Runs, type CreateEvalCompletionsRunDataSource, type CreateEvalJSONLRunDataSource, type EvalAPIError, type RunCreateResponse, type RunRetrieveResponse, type RunListResponse, type RunDeleteResponse, type RunCancelResponse, type RunCreateParams, type RunRetrieveParams, type RunListParams, type RunDeleteParams, type RunCancelParams, type RunListResponsesPage, } from "./runs/index.mjs";
//# sourceMappingURL=index.d.mts.map
landvex/node_modules/openai/resources/evals/index.d.mts.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../../src/resources/evals/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,KAAK,EACL,KAAK,0BAA0B,EAC/B,KAAK,qCAAqC,EAC1C,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,qBAAqB,GAC3B,oBAAgB;AACjB,OAAO,EACL,IAAI,EACJ,KAAK,kCAAkC,EACvC,KAAK,4BAA4B,EACjC,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,oBAAoB,GAC1B,yBAAqB"}
landvex/node_modules/openai/resources/evals/index.d.ts
Generated Vendored
+3
View File
@@ -0,0 +1,3 @@
export { Evals, type EvalCustomDataSourceConfig, type EvalStoredCompletionsDataSourceConfig, type EvalCreateResponse, type EvalRetrieveResponse, type EvalUpdateResponse, type EvalListResponse, type EvalDeleteResponse, type EvalCreateParams, type EvalUpdateParams, type EvalListParams, type EvalListResponsesPage, } from "./evals.js";
export { Runs, type CreateEvalCompletionsRunDataSource, type CreateEvalJSONLRunDataSource, type EvalAPIError, type RunCreateResponse, type RunRetrieveResponse, type RunListResponse, type RunDeleteResponse, type RunCancelResponse, type RunCreateParams, type RunRetrieveParams, type RunListParams, type RunDeleteParams, type RunCancelParams, type RunListResponsesPage, } from "./runs/index.js";
//# sourceMappingURL=index.d.ts.map
landvex/node_modules/openai/resources/evals/index.d.ts.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/resources/evals/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,KAAK,EACL,KAAK,0BAA0B,EAC/B,KAAK,qCAAqC,EAC1C,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,qBAAqB,GAC3B,mBAAgB;AACjB,OAAO,EACL,IAAI,EACJ,KAAK,kCAAkC,EACvC,KAAK,4BAA4B,EACjC,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,oBAAoB,GAC1B,wBAAqB"}
landvex/node_modules/openai/resources/evals/index.js
Generated Vendored
+9
View File
@@ -0,0 +1,9 @@
"use strict";
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
Object.defineProperty(exports, "__esModule", { value: true });
exports.Runs = exports.Evals = void 0;
var evals_1 = require("./evals.js");
Object.defineProperty(exports, "Evals", { enumerable: true, get: function () { return evals_1.Evals; } });
var index_1 = require("./runs/index.js");
Object.defineProperty(exports, "Runs", { enumerable: true, get: function () { return index_1.Runs; } });
//# sourceMappingURL=index.js.map
landvex/node_modules/openai/resources/evals/index.js.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/resources/evals/index.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,oCAaiB;AAZf,8FAAA,KAAK,OAAA;AAaP,yCAgBsB;AAfpB,6FAAA,IAAI,OAAA"}
landvex/node_modules/openai/resources/evals/index.mjs
Generated Vendored
+4
View File
@@ -0,0 +1,4 @@
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
export { Evals, } from "./evals.mjs";
export { Runs, } from "./runs/index.mjs";
//# sourceMappingURL=index.mjs.map
landvex/node_modules/openai/resources/evals/index.mjs.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../../src/resources/evals/index.ts"],"names":[],"mappings":"AAAA,sFAAsF;AAEtF,OAAO,EACL,KAAK,GAYN,oBAAgB;AACjB,OAAO,EACL,IAAI,GAeL,yBAAqB"}
landvex/node_modules/openai/resources/evals/runs.d.mts
Generated Vendored
+2
View File
@@ -0,0 +1,2 @@
export * from "./runs/index.mjs";
//# sourceMappingURL=runs.d.mts.map
landvex/node_modules/openai/resources/evals/runs.d.mts.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"runs.d.mts","sourceRoot":"","sources":["../../src/resources/evals/runs.ts"],"names":[],"mappings":"AAEA,iCAA6B"}
landvex/node_modules/openai/resources/evals/runs.d.ts
Generated Vendored
+2
View File
@@ -0,0 +1,2 @@
export * from "./runs/index.js";
//# sourceMappingURL=runs.d.ts.map
landvex/node_modules/openai/resources/evals/runs.d.ts.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"runs.d.ts","sourceRoot":"","sources":["../../src/resources/evals/runs.ts"],"names":[],"mappings":"AAEA,gCAA6B"}
landvex/node_modules/openai/resources/evals/runs.js
Generated Vendored
+6
View File
@@ -0,0 +1,6 @@
"use strict";
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
Object.defineProperty(exports, "__esModule", { value: true });
const tslib_1 = require("../../internal/tslib.js");
tslib_1.__exportStar(require("./runs/index.js"), exports);
//# sourceMappingURL=runs.js.map
landvex/node_modules/openai/resources/evals/runs.js.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"runs.js","sourceRoot":"","sources":["../../src/resources/evals/runs.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,0DAA6B"}
landvex/node_modules/openai/resources/evals/runs.mjs
Generated Vendored
+3
View File
@@ -0,0 +1,3 @@
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
export * from "./runs/index.mjs";
//# sourceMappingURL=runs.mjs.map
landvex/node_modules/openai/resources/evals/runs.mjs.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"runs.mjs","sourceRoot":"","sources":["../../src/resources/evals/runs.ts"],"names":[],"mappings":"AAAA,sFAAsF;AAEtF,iCAA6B"}
landvex/node_modules/openai/resources/evals/runs/index.d.mts
Generated Vendored
+3
View File
@@ -0,0 +1,3 @@
export { OutputItems, type OutputItemRetrieveResponse, type OutputItemListResponse, type OutputItemRetrieveParams, type OutputItemListParams, type OutputItemListResponsesPage, } from "./output-items.mjs";
export { Runs, type CreateEvalCompletionsRunDataSource, type CreateEvalJSONLRunDataSource, type EvalAPIError, type RunCreateResponse, type RunRetrieveResponse, type RunListResponse, type RunDeleteResponse, type RunCancelResponse, type RunCreateParams, type RunRetrieveParams, type RunListParams, type RunDeleteParams, type RunCancelParams, type RunListResponsesPage, } from "./runs.mjs";
//# sourceMappingURL=index.d.mts.map
landvex/node_modules/openai/resources/evals/runs/index.d.mts.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.d.mts","sourceRoot":"","sources":["../../../src/resources/evals/runs/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,WAAW,EACX,KAAK,0BAA0B,EAC/B,KAAK,sBAAsB,EAC3B,KAAK,wBAAwB,EAC7B,KAAK,oBAAoB,EACzB,KAAK,2BAA2B,GACjC,2BAAuB;AACxB,OAAO,EACL,IAAI,EACJ,KAAK,kCAAkC,EACvC,KAAK,4BAA4B,EACjC,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,oBAAoB,GAC1B,mBAAe"}
landvex/node_modules/openai/resources/evals/runs/index.d.ts
Generated Vendored
+3
View File
@@ -0,0 +1,3 @@
export { OutputItems, type OutputItemRetrieveResponse, type OutputItemListResponse, type OutputItemRetrieveParams, type OutputItemListParams, type OutputItemListResponsesPage, } from "./output-items.js";
export { Runs, type CreateEvalCompletionsRunDataSource, type CreateEvalJSONLRunDataSource, type EvalAPIError, type RunCreateResponse, type RunRetrieveResponse, type RunListResponse, type RunDeleteResponse, type RunCancelResponse, type RunCreateParams, type RunRetrieveParams, type RunListParams, type RunDeleteParams, type RunCancelParams, type RunListResponsesPage, } from "./runs.js";
//# sourceMappingURL=index.d.ts.map
landvex/node_modules/openai/resources/evals/runs/index.d.ts.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/resources/evals/runs/index.ts"],"names":[],"mappings":"AAEA,OAAO,EACL,WAAW,EACX,KAAK,0BAA0B,EAC/B,KAAK,sBAAsB,EAC3B,KAAK,wBAAwB,EAC7B,KAAK,oBAAoB,EACzB,KAAK,2BAA2B,GACjC,0BAAuB;AACxB,OAAO,EACL,IAAI,EACJ,KAAK,kCAAkC,EACvC,KAAK,4BAA4B,EACjC,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAClB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,oBAAoB,GAC1B,kBAAe"}
landvex/node_modules/openai/resources/evals/runs/index.js
Generated Vendored
+9
View File
@@ -0,0 +1,9 @@
"use strict";
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
Object.defineProperty(exports, "__esModule", { value: true });
exports.Runs = exports.OutputItems = void 0;
var output_items_1 = require("./output-items.js");
Object.defineProperty(exports, "OutputItems", { enumerable: true, get: function () { return output_items_1.OutputItems; } });
var runs_1 = require("./runs.js");
Object.defineProperty(exports, "Runs", { enumerable: true, get: function () { return runs_1.Runs; } });
//# sourceMappingURL=index.js.map
landvex/node_modules/openai/resources/evals/runs/index.js.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/resources/evals/runs/index.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,kDAOwB;AANtB,2GAAA,WAAW,OAAA;AAOb,kCAgBgB;AAfd,4FAAA,IAAI,OAAA"}
landvex/node_modules/openai/resources/evals/runs/index.mjs
Generated Vendored
+4
View File
@@ -0,0 +1,4 @@
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
export { OutputItems, } from "./output-items.mjs";
export { Runs, } from "./runs.mjs";
//# sourceMappingURL=index.mjs.map
landvex/node_modules/openai/resources/evals/runs/index.mjs.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"index.mjs","sourceRoot":"","sources":["../../../src/resources/evals/runs/index.ts"],"names":[],"mappings":"AAAA,sFAAsF;AAEtF,OAAO,EACL,WAAW,GAMZ,2BAAuB;AACxB,OAAO,EACL,IAAI,GAeL,mBAAe"}
landvex/node_modules/openai/resources/evals/runs/output-items.d.mts
Generated Vendored
+385
View File
@@ -0,0 +1,385 @@
import { APIResource } from "../../../core/resource.mjs";
import * as RunsAPI from "./runs.mjs";
import { APIPromise } from "../../../core/api-promise.mjs";
import { CursorPage, type CursorPageParams, PagePromise } from "../../../core/pagination.mjs";
import { RequestOptions } from "../../../internal/request-options.mjs";
/**
* Manage and run evals in the OpenAI platform.
*/
export declare class OutputItems extends APIResource {
/**
* Get an evaluation run output item by ID.
*/
retrieve(outputItemID: string, params: OutputItemRetrieveParams, options?: RequestOptions): APIPromise<OutputItemRetrieveResponse>;
/**
* Get a list of output items for an evaluation run.
*/
list(runID: string, params: OutputItemListParams, options?: RequestOptions): PagePromise<OutputItemListResponsesPage, OutputItemListResponse>;
}
export type OutputItemListResponsesPage = CursorPage<OutputItemListResponse>;
/**
* A schema representing an evaluation run output item.
*/
export interface OutputItemRetrieveResponse {
/**
* Unique identifier for the evaluation run output item.
*/
id: string;
/**
* Unix timestamp (in seconds) when the evaluation run was created.
*/
created_at: number;
/**
* Details of the input data source item.
*/
datasource_item: {
[key: string]: unknown;
};
/**
* The identifier for the data source item.
*/
datasource_item_id: number;
/**
* The identifier of the evaluation group.
*/
eval_id: string;
/**
* The type of the object. Always "eval.run.output_item".
*/
object: 'eval.run.output_item';
/**
* A list of grader results for this output item.
*/
results: Array<OutputItemRetrieveResponse.Result>;
/**
* The identifier of the evaluation run associated with this output item.
*/
run_id: string;
/**
* A sample containing the input and output of the evaluation run.
*/
sample: OutputItemRetrieveResponse.Sample;
/**
* The status of the evaluation run.
*/
status: string;
}
export declare namespace OutputItemRetrieveResponse {
/**
* A single grader result for an evaluation run output item.
*/
interface Result {
/**
* The name of the grader.
*/
name: string;
/**
* Whether the grader considered the output a pass.
*/
passed: boolean;
/**
* The numeric score produced by the grader.
*/
score: number;
/**
* Optional sample or intermediate data produced by the grader.
*/
sample?: {
[key: string]: unknown;
} | null;
/**
* The grader type (for example, "string-check-grader").
*/
type?: string;
[k: string]: unknown;
}
/**
* A sample containing the input and output of the evaluation run.
*/
interface Sample {
/**
* An object representing an error response from the Eval API.
*/
error: RunsAPI.EvalAPIError;
/**
* The reason why the sample generation was finished.
*/
finish_reason: string;
/**
* An array of input messages.
*/
input: Array<Sample.Input>;
/**
* The maximum number of tokens allowed for completion.
*/
max_completion_tokens: number;
/**
* The model used for generating the sample.
*/
model: string;
/**
* An array of output messages.
*/
output: Array<Sample.Output>;
/**
* The seed used for generating the sample.
*/
seed: number;
/**
* The sampling temperature used.
*/
temperature: number;
/**
* The top_p value used for sampling.
*/
top_p: number;
/**
* Token usage details for the sample.
*/
usage: Sample.Usage;
}
namespace Sample {
/**
* An input message.
*/
interface Input {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message sender (e.g., system, user, developer).
*/
role: string;
}
interface Output {
/**
* The content of the message.
*/
content?: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role?: string;
}
/**
* Token usage details for the sample.
*/
interface Usage {
/**
* The number of tokens retrieved from cache.
*/
cached_tokens: number;
/**
* The number of completion tokens generated.
*/
completion_tokens: number;
/**
* The number of prompt tokens used.
*/
prompt_tokens: number;
/**
* The total number of tokens used.
*/
total_tokens: number;
}
}
}
/**
* A schema representing an evaluation run output item.
*/
export interface OutputItemListResponse {
/**
* Unique identifier for the evaluation run output item.
*/
id: string;
/**
* Unix timestamp (in seconds) when the evaluation run was created.
*/
created_at: number;
/**
* Details of the input data source item.
*/
datasource_item: {
[key: string]: unknown;
};
/**
* The identifier for the data source item.
*/
datasource_item_id: number;
/**
* The identifier of the evaluation group.
*/
eval_id: string;
/**
* The type of the object. Always "eval.run.output_item".
*/
object: 'eval.run.output_item';
/**
* A list of grader results for this output item.
*/
results: Array<OutputItemListResponse.Result>;
/**
* The identifier of the evaluation run associated with this output item.
*/
run_id: string;
/**
* A sample containing the input and output of the evaluation run.
*/
sample: OutputItemListResponse.Sample;
/**
* The status of the evaluation run.
*/
status: string;
}
export declare namespace OutputItemListResponse {
/**
* A single grader result for an evaluation run output item.
*/
interface Result {
/**
* The name of the grader.
*/
name: string;
/**
* Whether the grader considered the output a pass.
*/
passed: boolean;
/**
* The numeric score produced by the grader.
*/
score: number;
/**
* Optional sample or intermediate data produced by the grader.
*/
sample?: {
[key: string]: unknown;
} | null;
/**
* The grader type (for example, "string-check-grader").
*/
type?: string;
[k: string]: unknown;
}
/**
* A sample containing the input and output of the evaluation run.
*/
interface Sample {
/**
* An object representing an error response from the Eval API.
*/
error: RunsAPI.EvalAPIError;
/**
* The reason why the sample generation was finished.
*/
finish_reason: string;
/**
* An array of input messages.
*/
input: Array<Sample.Input>;
/**
* The maximum number of tokens allowed for completion.
*/
max_completion_tokens: number;
/**
* The model used for generating the sample.
*/
model: string;
/**
* An array of output messages.
*/
output: Array<Sample.Output>;
/**
* The seed used for generating the sample.
*/
seed: number;
/**
* The sampling temperature used.
*/
temperature: number;
/**
* The top_p value used for sampling.
*/
top_p: number;
/**
* Token usage details for the sample.
*/
usage: Sample.Usage;
}
namespace Sample {
/**
* An input message.
*/
interface Input {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message sender (e.g., system, user, developer).
*/
role: string;
}
interface Output {
/**
* The content of the message.
*/
content?: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role?: string;
}
/**
* Token usage details for the sample.
*/
interface Usage {
/**
* The number of tokens retrieved from cache.
*/
cached_tokens: number;
/**
* The number of completion tokens generated.
*/
completion_tokens: number;
/**
* The number of prompt tokens used.
*/
prompt_tokens: number;
/**
* The total number of tokens used.
*/
total_tokens: number;
}
}
}
export interface OutputItemRetrieveParams {
/**
* The ID of the evaluation to retrieve runs for.
*/
eval_id: string;
/**
* The ID of the run to retrieve.
*/
run_id: string;
}
export interface OutputItemListParams extends CursorPageParams {
/**
* Path param: The ID of the evaluation to retrieve runs for.
*/
eval_id: string;
/**
* Query param: Sort order for output items by timestamp. Use `asc` for ascending
* order or `desc` for descending order. Defaults to `asc`.
*/
order?: 'asc' | 'desc';
/**
* Query param: Filter output items by status. Use `failed` to filter by failed
* output items or `pass` to filter by passed output items.
*/
status?: 'fail' | 'pass';
}
export declare namespace OutputItems {
export { type OutputItemRetrieveResponse as OutputItemRetrieveResponse, type OutputItemListResponse as OutputItemListResponse, type OutputItemListResponsesPage as OutputItemListResponsesPage, type OutputItemRetrieveParams as OutputItemRetrieveParams, type OutputItemListParams as OutputItemListParams, };
}
//# sourceMappingURL=output-items.d.mts.map
landvex/node_modules/openai/resources/evals/runs/output-items.d.mts.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"output-items.d.mts","sourceRoot":"","sources":["../../../src/resources/evals/runs/output-items.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,mCAA+B;AACrD,OAAO,KAAK,OAAO,mBAAe;AAClC,OAAO,EAAE,UAAU,EAAE,sCAAkC;AACvD,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,WAAW,EAAE,qCAAiC;AAC1F,OAAO,EAAE,cAAc,EAAE,8CAA0C;AAGnE;;GAEG;AACH,qBAAa,WAAY,SAAQ,WAAW;IAC1C;;OAEG;IACH,QAAQ,CACN,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,wBAAwB,EAChC,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,0BAA0B,CAAC;IAQzC;;OAEG;IACH,IAAI,CACF,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,oBAAoB,EAC5B,OAAO,CAAC,EAAE,cAAc,GACvB,WAAW,CAAC,2BAA2B,EAAE,sBAAsB,CAAC;CAQpE;AAED,MAAM,MAAM,2BAA2B,GAAG,UAAU,CAAC,sBAAsB,CAAC,CAAC;AAE7E;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,eAAe,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IAE5C;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,MAAM,EAAE,sBAAsB,CAAC;IAE/B;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,0BAA0B,CAAC,MAAM,CAAC,CAAC;IAElD;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,EAAE,0BAA0B,CAAC,MAAM,CAAC;IAE1C;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,yBAAiB,0BAA0B,CAAC;IAC1C;;OAEG;IACH,UAAiB,MAAM;QACrB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,MAAM,EAAE,OAAO,CAAC;QAEhB;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,MAAM,CAAC,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;SAAE,GAAG,IAAI,CAAC;QAE3C;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;KACtB;IAED;;OAEG;IACH,UAAiB,MAAM;QACrB;;WAEG;QACH,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC;QAE5B;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE3B;;WAEG;QACH,qBAAqB,EAAE,MAAM,CAAC;QAE9B;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE7B;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;KACrB;IAED,UAAiB,MAAM,CAAC;QACtB;;WAEG;QACH,UAAiB,KAAK;YACpB;;eAEG;YACH,OAAO,EAAE,MAAM,CAAC;YAEhB;;eAEG;YACH,IAAI,EAAE,MAAM,CAAC;SACd;QAED,UAAiB,MAAM;YACrB;;eAEG;YACH,OAAO,CAAC,EAAE,MAAM,CAAC;YAEjB;;eAEG;YACH,IAAI,CAAC,EAAE,MAAM,CAAC;SACf;QAED;;WAEG;QACH,UAAiB,KAAK;YACpB;;eAEG;YACH,aAAa,EAAE,MAAM,CAAC;YAEtB;;eAEG;YACH,iBAAiB,EAAE,MAAM,CAAC;YAE1B;;eAEG;YACH,aAAa,EAAE,MAAM,CAAC;YAEtB;;eAEG;YACH,YAAY,EAAE,MAAM,CAAC;SACtB;KACF;CACF;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,eAAe,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IAE5C;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,MAAM,EAAE,sBAAsB,CAAC;IAE/B;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAE9C;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,EAAE,sBAAsB,CAAC,MAAM,CAAC;IAEtC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,yBAAiB,sBAAsB,CAAC;IACtC;;OAEG;IACH,UAAiB,MAAM;QACrB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,MAAM,EAAE,OAAO,CAAC;QAEhB;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,MAAM,CAAC,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;SAAE,GAAG,IAAI,CAAC;QAE3C;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;KACtB;IAED;;OAEG;IACH,UAAiB,MAAM;QACrB;;WAEG;QACH,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC;QAE5B;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE3B;;WAEG;QACH,qBAAqB,EAAE,MAAM,CAAC;QAE9B;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE7B;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;KACrB;IAED,UAAiB,MAAM,CAAC;QACtB;;WAEG;QACH,UAAiB,KAAK;YACpB;;eAEG;YACH,OAAO,EAAE,MAAM,CAAC;YAEhB;;eAEG;YACH,IAAI,EAAE,MAAM,CAAC;SACd;QAED,UAAiB,MAAM;YACrB;;eAEG;YACH,OAAO,CAAC,EAAE,MAAM,CAAC;YAEjB;;eAEG;YACH,IAAI,CAAC,EAAE,MAAM,CAAC;SACf;QAED;;WAEG;QACH,UAAiB,KAAK;YACpB;;eAEG;YACH,aAAa,EAAE,MAAM,CAAC;YAEtB;;eAEG;YACH,iBAAiB,EAAE,MAAM,CAAC;YAE1B;;eAEG;YACH,aAAa,EAAE,MAAM,CAAC;YAEtB;;eAEG;YACH,YAAY,EAAE,MAAM,CAAC;SACtB;KACF;CACF;AAED,MAAM,WAAW,wBAAwB;IACvC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,oBAAqB,SAAQ,gBAAgB;IAC5D;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IAEvB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CAC1B;AAED,MAAM,CAAC,OAAO,WAAW,WAAW,CAAC;IACnC,OAAO,EACL,KAAK,0BAA0B,IAAI,0BAA0B,EAC7D,KAAK,sBAAsB,IAAI,sBAAsB,EACrD,KAAK,2BAA2B,IAAI,2BAA2B,EAC/D,KAAK,wBAAwB,IAAI,wBAAwB,EACzD,KAAK,oBAAoB,IAAI,oBAAoB,GAClD,CAAC;CACH"}
landvex/node_modules/openai/resources/evals/runs/output-items.d.ts
Generated Vendored
+385
View File
@@ -0,0 +1,385 @@
import { APIResource } from "../../../core/resource.js";
import * as RunsAPI from "./runs.js";
import { APIPromise } from "../../../core/api-promise.js";
import { CursorPage, type CursorPageParams, PagePromise } from "../../../core/pagination.js";
import { RequestOptions } from "../../../internal/request-options.js";
/**
* Manage and run evals in the OpenAI platform.
*/
export declare class OutputItems extends APIResource {
/**
* Get an evaluation run output item by ID.
*/
retrieve(outputItemID: string, params: OutputItemRetrieveParams, options?: RequestOptions): APIPromise<OutputItemRetrieveResponse>;
/**
* Get a list of output items for an evaluation run.
*/
list(runID: string, params: OutputItemListParams, options?: RequestOptions): PagePromise<OutputItemListResponsesPage, OutputItemListResponse>;
}
export type OutputItemListResponsesPage = CursorPage<OutputItemListResponse>;
/**
* A schema representing an evaluation run output item.
*/
export interface OutputItemRetrieveResponse {
/**
* Unique identifier for the evaluation run output item.
*/
id: string;
/**
* Unix timestamp (in seconds) when the evaluation run was created.
*/
created_at: number;
/**
* Details of the input data source item.
*/
datasource_item: {
[key: string]: unknown;
};
/**
* The identifier for the data source item.
*/
datasource_item_id: number;
/**
* The identifier of the evaluation group.
*/
eval_id: string;
/**
* The type of the object. Always "eval.run.output_item".
*/
object: 'eval.run.output_item';
/**
* A list of grader results for this output item.
*/
results: Array<OutputItemRetrieveResponse.Result>;
/**
* The identifier of the evaluation run associated with this output item.
*/
run_id: string;
/**
* A sample containing the input and output of the evaluation run.
*/
sample: OutputItemRetrieveResponse.Sample;
/**
* The status of the evaluation run.
*/
status: string;
}
export declare namespace OutputItemRetrieveResponse {
/**
* A single grader result for an evaluation run output item.
*/
interface Result {
/**
* The name of the grader.
*/
name: string;
/**
* Whether the grader considered the output a pass.
*/
passed: boolean;
/**
* The numeric score produced by the grader.
*/
score: number;
/**
* Optional sample or intermediate data produced by the grader.
*/
sample?: {
[key: string]: unknown;
} | null;
/**
* The grader type (for example, "string-check-grader").
*/
type?: string;
[k: string]: unknown;
}
/**
* A sample containing the input and output of the evaluation run.
*/
interface Sample {
/**
* An object representing an error response from the Eval API.
*/
error: RunsAPI.EvalAPIError;
/**
* The reason why the sample generation was finished.
*/
finish_reason: string;
/**
* An array of input messages.
*/
input: Array<Sample.Input>;
/**
* The maximum number of tokens allowed for completion.
*/
max_completion_tokens: number;
/**
* The model used for generating the sample.
*/
model: string;
/**
* An array of output messages.
*/
output: Array<Sample.Output>;
/**
* The seed used for generating the sample.
*/
seed: number;
/**
* The sampling temperature used.
*/
temperature: number;
/**
* The top_p value used for sampling.
*/
top_p: number;
/**
* Token usage details for the sample.
*/
usage: Sample.Usage;
}
namespace Sample {
/**
* An input message.
*/
interface Input {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message sender (e.g., system, user, developer).
*/
role: string;
}
interface Output {
/**
* The content of the message.
*/
content?: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role?: string;
}
/**
* Token usage details for the sample.
*/
interface Usage {
/**
* The number of tokens retrieved from cache.
*/
cached_tokens: number;
/**
* The number of completion tokens generated.
*/
completion_tokens: number;
/**
* The number of prompt tokens used.
*/
prompt_tokens: number;
/**
* The total number of tokens used.
*/
total_tokens: number;
}
}
}
/**
* A schema representing an evaluation run output item.
*/
export interface OutputItemListResponse {
/**
* Unique identifier for the evaluation run output item.
*/
id: string;
/**
* Unix timestamp (in seconds) when the evaluation run was created.
*/
created_at: number;
/**
* Details of the input data source item.
*/
datasource_item: {
[key: string]: unknown;
};
/**
* The identifier for the data source item.
*/
datasource_item_id: number;
/**
* The identifier of the evaluation group.
*/
eval_id: string;
/**
* The type of the object. Always "eval.run.output_item".
*/
object: 'eval.run.output_item';
/**
* A list of grader results for this output item.
*/
results: Array<OutputItemListResponse.Result>;
/**
* The identifier of the evaluation run associated with this output item.
*/
run_id: string;
/**
* A sample containing the input and output of the evaluation run.
*/
sample: OutputItemListResponse.Sample;
/**
* The status of the evaluation run.
*/
status: string;
}
export declare namespace OutputItemListResponse {
/**
* A single grader result for an evaluation run output item.
*/
interface Result {
/**
* The name of the grader.
*/
name: string;
/**
* Whether the grader considered the output a pass.
*/
passed: boolean;
/**
* The numeric score produced by the grader.
*/
score: number;
/**
* Optional sample or intermediate data produced by the grader.
*/
sample?: {
[key: string]: unknown;
} | null;
/**
* The grader type (for example, "string-check-grader").
*/
type?: string;
[k: string]: unknown;
}
/**
* A sample containing the input and output of the evaluation run.
*/
interface Sample {
/**
* An object representing an error response from the Eval API.
*/
error: RunsAPI.EvalAPIError;
/**
* The reason why the sample generation was finished.
*/
finish_reason: string;
/**
* An array of input messages.
*/
input: Array<Sample.Input>;
/**
* The maximum number of tokens allowed for completion.
*/
max_completion_tokens: number;
/**
* The model used for generating the sample.
*/
model: string;
/**
* An array of output messages.
*/
output: Array<Sample.Output>;
/**
* The seed used for generating the sample.
*/
seed: number;
/**
* The sampling temperature used.
*/
temperature: number;
/**
* The top_p value used for sampling.
*/
top_p: number;
/**
* Token usage details for the sample.
*/
usage: Sample.Usage;
}
namespace Sample {
/**
* An input message.
*/
interface Input {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message sender (e.g., system, user, developer).
*/
role: string;
}
interface Output {
/**
* The content of the message.
*/
content?: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role?: string;
}
/**
* Token usage details for the sample.
*/
interface Usage {
/**
* The number of tokens retrieved from cache.
*/
cached_tokens: number;
/**
* The number of completion tokens generated.
*/
completion_tokens: number;
/**
* The number of prompt tokens used.
*/
prompt_tokens: number;
/**
* The total number of tokens used.
*/
total_tokens: number;
}
}
}
export interface OutputItemRetrieveParams {
/**
* The ID of the evaluation to retrieve runs for.
*/
eval_id: string;
/**
* The ID of the run to retrieve.
*/
run_id: string;
}
export interface OutputItemListParams extends CursorPageParams {
/**
* Path param: The ID of the evaluation to retrieve runs for.
*/
eval_id: string;
/**
* Query param: Sort order for output items by timestamp. Use `asc` for ascending
* order or `desc` for descending order. Defaults to `asc`.
*/
order?: 'asc' | 'desc';
/**
* Query param: Filter output items by status. Use `failed` to filter by failed
* output items or `pass` to filter by passed output items.
*/
status?: 'fail' | 'pass';
}
export declare namespace OutputItems {
export { type OutputItemRetrieveResponse as OutputItemRetrieveResponse, type OutputItemListResponse as OutputItemListResponse, type OutputItemListResponsesPage as OutputItemListResponsesPage, type OutputItemRetrieveParams as OutputItemRetrieveParams, type OutputItemListParams as OutputItemListParams, };
}
//# sourceMappingURL=output-items.d.ts.map
landvex/node_modules/openai/resources/evals/runs/output-items.d.ts.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"output-items.d.ts","sourceRoot":"","sources":["../../../src/resources/evals/runs/output-items.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,WAAW,EAAE,kCAA+B;AACrD,OAAO,KAAK,OAAO,kBAAe;AAClC,OAAO,EAAE,UAAU,EAAE,qCAAkC;AACvD,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,WAAW,EAAE,oCAAiC;AAC1F,OAAO,EAAE,cAAc,EAAE,6CAA0C;AAGnE;;GAEG;AACH,qBAAa,WAAY,SAAQ,WAAW;IAC1C;;OAEG;IACH,QAAQ,CACN,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,wBAAwB,EAChC,OAAO,CAAC,EAAE,cAAc,GACvB,UAAU,CAAC,0BAA0B,CAAC;IAQzC;;OAEG;IACH,IAAI,CACF,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,oBAAoB,EAC5B,OAAO,CAAC,EAAE,cAAc,GACvB,WAAW,CAAC,2BAA2B,EAAE,sBAAsB,CAAC;CAQpE;AAED,MAAM,MAAM,2BAA2B,GAAG,UAAU,CAAC,sBAAsB,CAAC,CAAC;AAE7E;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,eAAe,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IAE5C;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,MAAM,EAAE,sBAAsB,CAAC;IAE/B;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,0BAA0B,CAAC,MAAM,CAAC,CAAC;IAElD;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,EAAE,0BAA0B,CAAC,MAAM,CAAC;IAE1C;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,yBAAiB,0BAA0B,CAAC;IAC1C;;OAEG;IACH,UAAiB,MAAM;QACrB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,MAAM,EAAE,OAAO,CAAC;QAEhB;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,MAAM,CAAC,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;SAAE,GAAG,IAAI,CAAC;QAE3C;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;KACtB;IAED;;OAEG;IACH,UAAiB,MAAM;QACrB;;WAEG;QACH,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC;QAE5B;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE3B;;WAEG;QACH,qBAAqB,EAAE,MAAM,CAAC;QAE9B;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE7B;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;KACrB;IAED,UAAiB,MAAM,CAAC;QACtB;;WAEG;QACH,UAAiB,KAAK;YACpB;;eAEG;YACH,OAAO,EAAE,MAAM,CAAC;YAEhB;;eAEG;YACH,IAAI,EAAE,MAAM,CAAC;SACd;QAED,UAAiB,MAAM;YACrB;;eAEG;YACH,OAAO,CAAC,EAAE,MAAM,CAAC;YAEjB;;eAEG;YACH,IAAI,CAAC,EAAE,MAAM,CAAC;SACf;QAED;;WAEG;QACH,UAAiB,KAAK;YACpB;;eAEG;YACH,aAAa,EAAE,MAAM,CAAC;YAEtB;;eAEG;YACH,iBAAiB,EAAE,MAAM,CAAC;YAE1B;;eAEG;YACH,aAAa,EAAE,MAAM,CAAC;YAEtB;;eAEG;YACH,YAAY,EAAE,MAAM,CAAC;SACtB;KACF;CACF;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,eAAe,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC;IAE5C;;OAEG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,MAAM,EAAE,sBAAsB,CAAC;IAE/B;;OAEG;IACH,OAAO,EAAE,KAAK,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAE9C;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,MAAM,EAAE,sBAAsB,CAAC,MAAM,CAAC;IAEtC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,yBAAiB,sBAAsB,CAAC;IACtC;;OAEG;IACH,UAAiB,MAAM;QACrB;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,MAAM,EAAE,OAAO,CAAC;QAEhB;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,MAAM,CAAC,EAAE;YAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;SAAE,GAAG,IAAI,CAAC;QAE3C;;WAEG;QACH,IAAI,CAAC,EAAE,MAAM,CAAC;QAEd,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;KACtB;IAED;;OAEG;IACH,UAAiB,MAAM;QACrB;;WAEG;QACH,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC;QAE5B;;WAEG;QACH,aAAa,EAAE,MAAM,CAAC;QAEtB;;WAEG;QACH,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE3B;;WAEG;QACH,qBAAqB,EAAE,MAAM,CAAC;QAE9B;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAE7B;;WAEG;QACH,IAAI,EAAE,MAAM,CAAC;QAEb;;WAEG;QACH,WAAW,EAAE,MAAM,CAAC;QAEpB;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC;QAEd;;WAEG;QACH,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;KACrB;IAED,UAAiB,MAAM,CAAC;QACtB;;WAEG;QACH,UAAiB,KAAK;YACpB;;eAEG;YACH,OAAO,EAAE,MAAM,CAAC;YAEhB;;eAEG;YACH,IAAI,EAAE,MAAM,CAAC;SACd;QAED,UAAiB,MAAM;YACrB;;eAEG;YACH,OAAO,CAAC,EAAE,MAAM,CAAC;YAEjB;;eAEG;YACH,IAAI,CAAC,EAAE,MAAM,CAAC;SACf;QAED;;WAEG;QACH,UAAiB,KAAK;YACpB;;eAEG;YACH,aAAa,EAAE,MAAM,CAAC;YAEtB;;eAEG;YACH,iBAAiB,EAAE,MAAM,CAAC;YAE1B;;eAEG;YACH,aAAa,EAAE,MAAM,CAAC;YAEtB;;eAEG;YACH,YAAY,EAAE,MAAM,CAAC;SACtB;KACF;CACF;AAED,MAAM,WAAW,wBAAwB;IACvC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,oBAAqB,SAAQ,gBAAgB;IAC5D;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IAEvB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CAC1B;AAED,MAAM,CAAC,OAAO,WAAW,WAAW,CAAC;IACnC,OAAO,EACL,KAAK,0BAA0B,IAAI,0BAA0B,EAC7D,KAAK,sBAAsB,IAAI,sBAAsB,EACrD,KAAK,2BAA2B,IAAI,2BAA2B,EAC/D,KAAK,wBAAwB,IAAI,wBAAwB,EACzD,KAAK,oBAAoB,IAAI,oBAAoB,GAClD,CAAC;CACH"}
landvex/node_modules/openai/resources/evals/runs/output-items.js
Generated Vendored
+31
View File
@@ -0,0 +1,31 @@
"use strict";
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
Object.defineProperty(exports, "__esModule", { value: true });
exports.OutputItems = void 0;
const resource_1 = require("../../../core/resource.js");
const pagination_1 = require("../../../core/pagination.js");
const path_1 = require("../../../internal/utils/path.js");
/**
* Manage and run evals in the OpenAI platform.
*/
class OutputItems extends resource_1.APIResource {
/**
* Get an evaluation run output item by ID.
*/
retrieve(outputItemID, params, options) {
const { eval_id, run_id } = params;
return this._client.get((0, path_1.path) `/evals/${eval_id}/runs/${run_id}/output_items/${outputItemID}`, {
...options,
__security: { bearerAuth: true },
});
}
/**
* Get a list of output items for an evaluation run.
*/
list(runID, params, options) {
const { eval_id, ...query } = params;
return this._client.getAPIList((0, path_1.path) `/evals/${eval_id}/runs/${runID}/output_items`, (pagination_1.CursorPage), { query, ...options, __security: { bearerAuth: true } });
}
}
exports.OutputItems = OutputItems;
//# sourceMappingURL=output-items.js.map
landvex/node_modules/openai/resources/evals/runs/output-items.js.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"output-items.js","sourceRoot":"","sources":["../../../src/resources/evals/runs/output-items.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;AAEtF,wDAAqD;AAGrD,4DAA0F;AAE1F,0DAAoD;AAEpD;;GAEG;AACH,MAAa,WAAY,SAAQ,sBAAW;IAC1C;;OAEG;IACH,QAAQ,CACN,YAAoB,EACpB,MAAgC,EAChC,OAAwB;QAExB,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,UAAU,OAAO,SAAS,MAAM,iBAAiB,YAAY,EAAE,EAAE;YAC3F,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAI,CACF,KAAa,EACb,MAA4B,EAC5B,OAAwB;QAExB,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,EAAE,GAAG,MAAM,CAAC;QACrC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAC5B,IAAA,WAAI,EAAA,UAAU,OAAO,SAAS,KAAK,eAAe,EAClD,CAAA,uBAAkC,CAAA,EAClC,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CACxD,CAAC;IACJ,CAAC;CACF;AA/BD,kCA+BC"}
landvex/node_modules/openai/resources/evals/runs/output-items.mjs
Generated Vendored
+27
View File
@@ -0,0 +1,27 @@
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
import { APIResource } from "../../../core/resource.mjs";
import { CursorPage } from "../../../core/pagination.mjs";
import { path } from "../../../internal/utils/path.mjs";
/**
* Manage and run evals in the OpenAI platform.
*/
export class OutputItems extends APIResource {
/**
* Get an evaluation run output item by ID.
*/
retrieve(outputItemID, params, options) {
const { eval_id, run_id } = params;
return this._client.get(path `/evals/${eval_id}/runs/${run_id}/output_items/${outputItemID}`, {
...options,
__security: { bearerAuth: true },
});
}
/**
* Get a list of output items for an evaluation run.
*/
list(runID, params, options) {
const { eval_id, ...query } = params;
return this._client.getAPIList(path `/evals/${eval_id}/runs/${runID}/output_items`, (CursorPage), { query, ...options, __security: { bearerAuth: true } });
}
}
//# sourceMappingURL=output-items.mjs.map
landvex/node_modules/openai/resources/evals/runs/output-items.mjs.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"output-items.mjs","sourceRoot":"","sources":["../../../src/resources/evals/runs/output-items.ts"],"names":[],"mappings":"AAAA,sFAAsF;AAEtF,OAAO,EAAE,WAAW,EAAE,mCAA+B;AAGrD,OAAO,EAAE,UAAU,EAAsC,qCAAiC;AAE1F,OAAO,EAAE,IAAI,EAAE,yCAAqC;AAEpD;;GAEG;AACH,MAAM,OAAO,WAAY,SAAQ,WAAW;IAC1C;;OAEG;IACH,QAAQ,CACN,YAAoB,EACpB,MAAgC,EAChC,OAAwB;QAExB,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,UAAU,OAAO,SAAS,MAAM,iBAAiB,YAAY,EAAE,EAAE;YAC3F,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAI,CACF,KAAa,EACb,MAA4B,EAC5B,OAAwB;QAExB,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,EAAE,GAAG,MAAM,CAAC;QACrC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAC5B,IAAI,CAAA,UAAU,OAAO,SAAS,KAAK,eAAe,EAClD,CAAA,UAAkC,CAAA,EAClC,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CACxD,CAAC;IACJ,CAAC;CACF"}
landvex/node_modules/openai/resources/evals/runs/runs.d.mts
Generated Vendored
+2388
View File
@@ -0,0 +1,2388 @@
import { APIResource } from "../../../core/resource.mjs";
import * as Shared from "../../shared.mjs";
import * as GraderModelsAPI from "../../graders/grader-models.mjs";
import * as ResponsesAPI from "../../responses/responses.mjs";
import * as CompletionsAPI from "../../chat/completions/completions.mjs";
import * as OutputItemsAPI from "./output-items.mjs";
import { OutputItemListParams, OutputItemListResponse, OutputItemListResponsesPage, OutputItemRetrieveParams, OutputItemRetrieveResponse, OutputItems } from "./output-items.mjs";
import { APIPromise } from "../../../core/api-promise.mjs";
import { CursorPage, type CursorPageParams, PagePromise } from "../../../core/pagination.mjs";
import { RequestOptions } from "../../../internal/request-options.mjs";
/**
* Manage and run evals in the OpenAI platform.
*/
export declare class Runs extends APIResource {
outputItems: OutputItemsAPI.OutputItems;
/**
* Kicks off a new run for a given evaluation, specifying the data source, and what
* model configuration to use to test. The datasource will be validated against the
* schema specified in the config of the evaluation.
*/
create(evalID: string, body: RunCreateParams, options?: RequestOptions): APIPromise<RunCreateResponse>;
/**
* Get an evaluation run by ID.
*/
retrieve(runID: string, params: RunRetrieveParams, options?: RequestOptions): APIPromise<RunRetrieveResponse>;
/**
* Get a list of runs for an evaluation.
*/
list(evalID: string, query?: RunListParams | null | undefined, options?: RequestOptions): PagePromise<RunListResponsesPage, RunListResponse>;
/**
* Delete an eval run.
*/
delete(runID: string, params: RunDeleteParams, options?: RequestOptions): APIPromise<RunDeleteResponse>;
/**
* Cancel an ongoing evaluation run.
*/
cancel(runID: string, params: RunCancelParams, options?: RequestOptions): APIPromise<RunCancelResponse>;
}
export type RunListResponsesPage = CursorPage<RunListResponse>;
/**
* A CompletionsRunDataSource object describing a model sampling configuration.
*/
export interface CreateEvalCompletionsRunDataSource {
/**
* Determines what populates the `item` namespace in this run's data source.
*/
source: CreateEvalCompletionsRunDataSource.FileContent | CreateEvalCompletionsRunDataSource.FileID | CreateEvalCompletionsRunDataSource.StoredCompletions;
/**
* The type of run data source. Always `completions`.
*/
type: 'completions';
/**
* Used when sampling from a model. Dictates the structure of the messages passed
* into the model. Can either be a reference to a prebuilt trajectory (ie,
* `item.input_trajectory`), or a template with variable references to the `item`
* namespace.
*/
input_messages?: CreateEvalCompletionsRunDataSource.Template | CreateEvalCompletionsRunDataSource.ItemReference;
/**
* The name of the model to use for generating completions (e.g. "o3-mini").
*/
model?: string;
sampling_params?: CreateEvalCompletionsRunDataSource.SamplingParams;
}
export declare namespace CreateEvalCompletionsRunDataSource {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
/**
* A StoredCompletionsRunDataSource configuration describing a set of filters
*/
interface StoredCompletions {
/**
* The type of source. Always `stored_completions`.
*/
type: 'stored_completions';
/**
* An optional Unix timestamp to filter items created after this time.
*/
created_after?: number | null;
/**
* An optional Unix timestamp to filter items created before this time.
*/
created_before?: number | null;
/**
* An optional maximum number of items to return.
*/
limit?: number | null;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
/**
* An optional model to filter by (e.g., 'gpt-4o').
*/
model?: string | null;
}
interface Template {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
template: Array<ResponsesAPI.EasyInputMessage | Template.EvalItem>;
/**
* The type of input messages. Always `template`.
*/
type: 'template';
}
namespace Template {
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
interface ItemReference {
/**
* A reference to a variable in the `item` namespace. Ie, "item.input_trajectory"
*/
item_reference: string;
/**
* The type of input messages. Always `item_reference`.
*/
type: 'item_reference';
}
interface SamplingParams {
/**
* The maximum number of tokens in the generated output.
*/
max_completion_tokens?: number;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* An object specifying the format that the model must output.
*
* Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured
* Outputs which ensures the model will match your supplied JSON schema. Learn more
* in the
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
*
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
* ensures the message the model generates is valid JSON. Using `json_schema` is
* preferred for models that support it.
*/
response_format?: Shared.ResponseFormatText | Shared.ResponseFormatJSONSchema | Shared.ResponseFormatJSONObject;
/**
* A seed value to initialize the randomness, during sampling.
*/
seed?: number;
/**
* A higher temperature increases randomness in the outputs.
*/
temperature?: number;
/**
* A list of tools the model may call. Currently, only functions are supported as a
* tool. Use this to provide a list of functions the model may generate JSON inputs
* for. A max of 128 functions are supported.
*/
tools?: Array<CompletionsAPI.ChatCompletionFunctionTool>;
/**
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
*/
top_p?: number;
}
}
/**
* A JsonlRunDataSource object with that specifies a JSONL file that matches the
* eval
*/
export interface CreateEvalJSONLRunDataSource {
/**
* Determines what populates the `item` namespace in the data source.
*/
source: CreateEvalJSONLRunDataSource.FileContent | CreateEvalJSONLRunDataSource.FileID;
/**
* The type of data source. Always `jsonl`.
*/
type: 'jsonl';
}
export declare namespace CreateEvalJSONLRunDataSource {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
}
/**
* An object representing an error response from the Eval API.
*/
export interface EvalAPIError {
/**
* The error code.
*/
code: string;
/**
* The error message.
*/
message: string;
}
/**
* A schema representing an evaluation run.
*/
export interface RunCreateResponse {
/**
* Unique identifier for the evaluation run.
*/
id: string;
/**
* Unix timestamp (in seconds) when the evaluation run was created.
*/
created_at: number;
/**
* Information about the run's data source.
*/
data_source: CreateEvalJSONLRunDataSource | CreateEvalCompletionsRunDataSource | RunCreateResponse.Responses;
/**
* An object representing an error response from the Eval API.
*/
error: EvalAPIError;
/**
* The identifier of the associated evaluation.
*/
eval_id: string;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The model that is evaluated, if applicable.
*/
model: string;
/**
* The name of the evaluation run.
*/
name: string;
/**
* The type of the object. Always "eval.run".
*/
object: 'eval.run';
/**
* Usage statistics for each model during the evaluation run.
*/
per_model_usage: Array<RunCreateResponse.PerModelUsage>;
/**
* Results per testing criteria applied during the evaluation run.
*/
per_testing_criteria_results: Array<RunCreateResponse.PerTestingCriteriaResult>;
/**
* The URL to the rendered evaluation run report on the UI dashboard.
*/
report_url: string;
/**
* Counters summarizing the outcomes of the evaluation run.
*/
result_counts: RunCreateResponse.ResultCounts;
/**
* The status of the evaluation run.
*/
status: string;
}
export declare namespace RunCreateResponse {
/**
* A ResponsesRunDataSource object describing a model sampling configuration.
*/
interface Responses {
/**
* Determines what populates the `item` namespace in this run's data source.
*/
source: Responses.FileContent | Responses.FileID | Responses.Responses;
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Used when sampling from a model. Dictates the structure of the messages passed
* into the model. Can either be a reference to a prebuilt trajectory (ie,
* `item.input_trajectory`), or a template with variable references to the `item`
* namespace.
*/
input_messages?: Responses.Template | Responses.ItemReference;
/**
* The name of the model to use for generating completions (e.g. "o3-mini").
*/
model?: string;
sampling_params?: Responses.SamplingParams;
}
namespace Responses {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
/**
* A EvalResponsesSource object describing a run data source configuration.
*/
interface Responses {
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Only include items created after this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_after?: number | null;
/**
* Only include items created before this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_before?: number | null;
/**
* Optional string to search the 'instructions' field. This is a query parameter
* used to select responses.
*/
instructions_search?: string | null;
/**
* Metadata filter for the responses. This is a query parameter used to select
* responses.
*/
metadata?: unknown | null;
/**
* The name of the model to find responses for. This is a query parameter used to
* select responses.
*/
model?: string | null;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* Sampling temperature. This is a query parameter used to select responses.
*/
temperature?: number | null;
/**
* List of tool names. This is a query parameter used to select responses.
*/
tools?: Array<string> | null;
/**
* Nucleus sampling parameter. This is a query parameter used to select responses.
*/
top_p?: number | null;
/**
* List of user identifiers. This is a query parameter used to select responses.
*/
users?: Array<string> | null;
}
interface Template {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
template: Array<Template.ChatMessage | Template.EvalItem>;
/**
* The type of input messages. Always `template`.
*/
type: 'template';
}
namespace Template {
interface ChatMessage {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role: string;
}
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
interface ItemReference {
/**
* A reference to a variable in the `item` namespace. Ie, "item.name"
*/
item_reference: string;
/**
* The type of input messages. Always `item_reference`.
*/
type: 'item_reference';
}
interface SamplingParams {
/**
* The maximum number of tokens in the generated output.
*/
max_completion_tokens?: number;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* A seed value to initialize the randomness, during sampling.
*/
seed?: number;
/**
* A higher temperature increases randomness in the outputs.
*/
temperature?: number;
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
text?: SamplingParams.Text;
/**
* An array of tools the model may call while generating a response. You can
* specify which tool to use by setting the `tool_choice` parameter.
*
* The two categories of tools you can provide the model are:
*
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
* capabilities, like
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
* Learn more about
* [built-in tools](https://platform.openai.com/docs/guides/tools).
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
* the model to call your own code. Learn more about
* [function calling](https://platform.openai.com/docs/guides/function-calling).
*/
tools?: Array<ResponsesAPI.Tool>;
/**
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
*/
top_p?: number;
}
namespace SamplingParams {
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
interface Text {
/**
* An object specifying the format that the model must output.
*
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
* ensures the model will match your supplied JSON schema. Learn more in the
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
*
* The default format is `{ "type": "text" }` with no additional options.
*
* **Not recommended for gpt-4o and newer models:**
*
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
* ensures the message the model generates is valid JSON. Using `json_schema` is
* preferred for models that support it.
*/
format?: ResponsesAPI.ResponseFormatTextConfig;
}
}
}
interface PerModelUsage {
/**
* The number of tokens retrieved from cache.
*/
cached_tokens: number;
/**
* The number of completion tokens generated.
*/
completion_tokens: number;
/**
* The number of invocations.
*/
invocation_count: number;
/**
* The name of the model.
*/
model_name: string;
/**
* The number of prompt tokens used.
*/
prompt_tokens: number;
/**
* The total number of tokens used.
*/
total_tokens: number;
}
interface PerTestingCriteriaResult {
/**
* Number of tests failed for this criteria.
*/
failed: number;
/**
* Number of tests passed for this criteria.
*/
passed: number;
/**
* A description of the testing criteria.
*/
testing_criteria: string;
}
/**
* Counters summarizing the outcomes of the evaluation run.
*/
interface ResultCounts {
/**
* Number of output items that resulted in an error.
*/
errored: number;
/**
* Number of output items that failed to pass the evaluation.
*/
failed: number;
/**
* Number of output items that passed the evaluation.
*/
passed: number;
/**
* Total number of executed output items.
*/
total: number;
}
}
/**
* A schema representing an evaluation run.
*/
export interface RunRetrieveResponse {
/**
* Unique identifier for the evaluation run.
*/
id: string;
/**
* Unix timestamp (in seconds) when the evaluation run was created.
*/
created_at: number;
/**
* Information about the run's data source.
*/
data_source: CreateEvalJSONLRunDataSource | CreateEvalCompletionsRunDataSource | RunRetrieveResponse.Responses;
/**
* An object representing an error response from the Eval API.
*/
error: EvalAPIError;
/**
* The identifier of the associated evaluation.
*/
eval_id: string;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The model that is evaluated, if applicable.
*/
model: string;
/**
* The name of the evaluation run.
*/
name: string;
/**
* The type of the object. Always "eval.run".
*/
object: 'eval.run';
/**
* Usage statistics for each model during the evaluation run.
*/
per_model_usage: Array<RunRetrieveResponse.PerModelUsage>;
/**
* Results per testing criteria applied during the evaluation run.
*/
per_testing_criteria_results: Array<RunRetrieveResponse.PerTestingCriteriaResult>;
/**
* The URL to the rendered evaluation run report on the UI dashboard.
*/
report_url: string;
/**
* Counters summarizing the outcomes of the evaluation run.
*/
result_counts: RunRetrieveResponse.ResultCounts;
/**
* The status of the evaluation run.
*/
status: string;
}
export declare namespace RunRetrieveResponse {
/**
* A ResponsesRunDataSource object describing a model sampling configuration.
*/
interface Responses {
/**
* Determines what populates the `item` namespace in this run's data source.
*/
source: Responses.FileContent | Responses.FileID | Responses.Responses;
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Used when sampling from a model. Dictates the structure of the messages passed
* into the model. Can either be a reference to a prebuilt trajectory (ie,
* `item.input_trajectory`), or a template with variable references to the `item`
* namespace.
*/
input_messages?: Responses.Template | Responses.ItemReference;
/**
* The name of the model to use for generating completions (e.g. "o3-mini").
*/
model?: string;
sampling_params?: Responses.SamplingParams;
}
namespace Responses {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
/**
* A EvalResponsesSource object describing a run data source configuration.
*/
interface Responses {
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Only include items created after this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_after?: number | null;
/**
* Only include items created before this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_before?: number | null;
/**
* Optional string to search the 'instructions' field. This is a query parameter
* used to select responses.
*/
instructions_search?: string | null;
/**
* Metadata filter for the responses. This is a query parameter used to select
* responses.
*/
metadata?: unknown | null;
/**
* The name of the model to find responses for. This is a query parameter used to
* select responses.
*/
model?: string | null;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* Sampling temperature. This is a query parameter used to select responses.
*/
temperature?: number | null;
/**
* List of tool names. This is a query parameter used to select responses.
*/
tools?: Array<string> | null;
/**
* Nucleus sampling parameter. This is a query parameter used to select responses.
*/
top_p?: number | null;
/**
* List of user identifiers. This is a query parameter used to select responses.
*/
users?: Array<string> | null;
}
interface Template {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
template: Array<Template.ChatMessage | Template.EvalItem>;
/**
* The type of input messages. Always `template`.
*/
type: 'template';
}
namespace Template {
interface ChatMessage {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role: string;
}
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
interface ItemReference {
/**
* A reference to a variable in the `item` namespace. Ie, "item.name"
*/
item_reference: string;
/**
* The type of input messages. Always `item_reference`.
*/
type: 'item_reference';
}
interface SamplingParams {
/**
* The maximum number of tokens in the generated output.
*/
max_completion_tokens?: number;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* A seed value to initialize the randomness, during sampling.
*/
seed?: number;
/**
* A higher temperature increases randomness in the outputs.
*/
temperature?: number;
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
text?: SamplingParams.Text;
/**
* An array of tools the model may call while generating a response. You can
* specify which tool to use by setting the `tool_choice` parameter.
*
* The two categories of tools you can provide the model are:
*
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
* capabilities, like
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
* Learn more about
* [built-in tools](https://platform.openai.com/docs/guides/tools).
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
* the model to call your own code. Learn more about
* [function calling](https://platform.openai.com/docs/guides/function-calling).
*/
tools?: Array<ResponsesAPI.Tool>;
/**
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
*/
top_p?: number;
}
namespace SamplingParams {
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
interface Text {
/**
* An object specifying the format that the model must output.
*
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
* ensures the model will match your supplied JSON schema. Learn more in the
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
*
* The default format is `{ "type": "text" }` with no additional options.
*
* **Not recommended for gpt-4o and newer models:**
*
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
* ensures the message the model generates is valid JSON. Using `json_schema` is
* preferred for models that support it.
*/
format?: ResponsesAPI.ResponseFormatTextConfig;
}
}
}
interface PerModelUsage {
/**
* The number of tokens retrieved from cache.
*/
cached_tokens: number;
/**
* The number of completion tokens generated.
*/
completion_tokens: number;
/**
* The number of invocations.
*/
invocation_count: number;
/**
* The name of the model.
*/
model_name: string;
/**
* The number of prompt tokens used.
*/
prompt_tokens: number;
/**
* The total number of tokens used.
*/
total_tokens: number;
}
interface PerTestingCriteriaResult {
/**
* Number of tests failed for this criteria.
*/
failed: number;
/**
* Number of tests passed for this criteria.
*/
passed: number;
/**
* A description of the testing criteria.
*/
testing_criteria: string;
}
/**
* Counters summarizing the outcomes of the evaluation run.
*/
interface ResultCounts {
/**
* Number of output items that resulted in an error.
*/
errored: number;
/**
* Number of output items that failed to pass the evaluation.
*/
failed: number;
/**
* Number of output items that passed the evaluation.
*/
passed: number;
/**
* Total number of executed output items.
*/
total: number;
}
}
/**
* A schema representing an evaluation run.
*/
export interface RunListResponse {
/**
* Unique identifier for the evaluation run.
*/
id: string;
/**
* Unix timestamp (in seconds) when the evaluation run was created.
*/
created_at: number;
/**
* Information about the run's data source.
*/
data_source: CreateEvalJSONLRunDataSource | CreateEvalCompletionsRunDataSource | RunListResponse.Responses;
/**
* An object representing an error response from the Eval API.
*/
error: EvalAPIError;
/**
* The identifier of the associated evaluation.
*/
eval_id: string;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The model that is evaluated, if applicable.
*/
model: string;
/**
* The name of the evaluation run.
*/
name: string;
/**
* The type of the object. Always "eval.run".
*/
object: 'eval.run';
/**
* Usage statistics for each model during the evaluation run.
*/
per_model_usage: Array<RunListResponse.PerModelUsage>;
/**
* Results per testing criteria applied during the evaluation run.
*/
per_testing_criteria_results: Array<RunListResponse.PerTestingCriteriaResult>;
/**
* The URL to the rendered evaluation run report on the UI dashboard.
*/
report_url: string;
/**
* Counters summarizing the outcomes of the evaluation run.
*/
result_counts: RunListResponse.ResultCounts;
/**
* The status of the evaluation run.
*/
status: string;
}
export declare namespace RunListResponse {
/**
* A ResponsesRunDataSource object describing a model sampling configuration.
*/
interface Responses {
/**
* Determines what populates the `item` namespace in this run's data source.
*/
source: Responses.FileContent | Responses.FileID | Responses.Responses;
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Used when sampling from a model. Dictates the structure of the messages passed
* into the model. Can either be a reference to a prebuilt trajectory (ie,
* `item.input_trajectory`), or a template with variable references to the `item`
* namespace.
*/
input_messages?: Responses.Template | Responses.ItemReference;
/**
* The name of the model to use for generating completions (e.g. "o3-mini").
*/
model?: string;
sampling_params?: Responses.SamplingParams;
}
namespace Responses {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
/**
* A EvalResponsesSource object describing a run data source configuration.
*/
interface Responses {
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Only include items created after this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_after?: number | null;
/**
* Only include items created before this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_before?: number | null;
/**
* Optional string to search the 'instructions' field. This is a query parameter
* used to select responses.
*/
instructions_search?: string | null;
/**
* Metadata filter for the responses. This is a query parameter used to select
* responses.
*/
metadata?: unknown | null;
/**
* The name of the model to find responses for. This is a query parameter used to
* select responses.
*/
model?: string | null;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* Sampling temperature. This is a query parameter used to select responses.
*/
temperature?: number | null;
/**
* List of tool names. This is a query parameter used to select responses.
*/
tools?: Array<string> | null;
/**
* Nucleus sampling parameter. This is a query parameter used to select responses.
*/
top_p?: number | null;
/**
* List of user identifiers. This is a query parameter used to select responses.
*/
users?: Array<string> | null;
}
interface Template {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
template: Array<Template.ChatMessage | Template.EvalItem>;
/**
* The type of input messages. Always `template`.
*/
type: 'template';
}
namespace Template {
interface ChatMessage {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role: string;
}
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
interface ItemReference {
/**
* A reference to a variable in the `item` namespace. Ie, "item.name"
*/
item_reference: string;
/**
* The type of input messages. Always `item_reference`.
*/
type: 'item_reference';
}
interface SamplingParams {
/**
* The maximum number of tokens in the generated output.
*/
max_completion_tokens?: number;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* A seed value to initialize the randomness, during sampling.
*/
seed?: number;
/**
* A higher temperature increases randomness in the outputs.
*/
temperature?: number;
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
text?: SamplingParams.Text;
/**
* An array of tools the model may call while generating a response. You can
* specify which tool to use by setting the `tool_choice` parameter.
*
* The two categories of tools you can provide the model are:
*
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
* capabilities, like
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
* Learn more about
* [built-in tools](https://platform.openai.com/docs/guides/tools).
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
* the model to call your own code. Learn more about
* [function calling](https://platform.openai.com/docs/guides/function-calling).
*/
tools?: Array<ResponsesAPI.Tool>;
/**
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
*/
top_p?: number;
}
namespace SamplingParams {
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
interface Text {
/**
* An object specifying the format that the model must output.
*
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
* ensures the model will match your supplied JSON schema. Learn more in the
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
*
* The default format is `{ "type": "text" }` with no additional options.
*
* **Not recommended for gpt-4o and newer models:**
*
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
* ensures the message the model generates is valid JSON. Using `json_schema` is
* preferred for models that support it.
*/
format?: ResponsesAPI.ResponseFormatTextConfig;
}
}
}
interface PerModelUsage {
/**
* The number of tokens retrieved from cache.
*/
cached_tokens: number;
/**
* The number of completion tokens generated.
*/
completion_tokens: number;
/**
* The number of invocations.
*/
invocation_count: number;
/**
* The name of the model.
*/
model_name: string;
/**
* The number of prompt tokens used.
*/
prompt_tokens: number;
/**
* The total number of tokens used.
*/
total_tokens: number;
}
interface PerTestingCriteriaResult {
/**
* Number of tests failed for this criteria.
*/
failed: number;
/**
* Number of tests passed for this criteria.
*/
passed: number;
/**
* A description of the testing criteria.
*/
testing_criteria: string;
}
/**
* Counters summarizing the outcomes of the evaluation run.
*/
interface ResultCounts {
/**
* Number of output items that resulted in an error.
*/
errored: number;
/**
* Number of output items that failed to pass the evaluation.
*/
failed: number;
/**
* Number of output items that passed the evaluation.
*/
passed: number;
/**
* Total number of executed output items.
*/
total: number;
}
}
export interface RunDeleteResponse {
deleted?: boolean;
object?: string;
run_id?: string;
}
/**
* A schema representing an evaluation run.
*/
export interface RunCancelResponse {
/**
* Unique identifier for the evaluation run.
*/
id: string;
/**
* Unix timestamp (in seconds) when the evaluation run was created.
*/
created_at: number;
/**
* Information about the run's data source.
*/
data_source: CreateEvalJSONLRunDataSource | CreateEvalCompletionsRunDataSource | RunCancelResponse.Responses;
/**
* An object representing an error response from the Eval API.
*/
error: EvalAPIError;
/**
* The identifier of the associated evaluation.
*/
eval_id: string;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The model that is evaluated, if applicable.
*/
model: string;
/**
* The name of the evaluation run.
*/
name: string;
/**
* The type of the object. Always "eval.run".
*/
object: 'eval.run';
/**
* Usage statistics for each model during the evaluation run.
*/
per_model_usage: Array<RunCancelResponse.PerModelUsage>;
/**
* Results per testing criteria applied during the evaluation run.
*/
per_testing_criteria_results: Array<RunCancelResponse.PerTestingCriteriaResult>;
/**
* The URL to the rendered evaluation run report on the UI dashboard.
*/
report_url: string;
/**
* Counters summarizing the outcomes of the evaluation run.
*/
result_counts: RunCancelResponse.ResultCounts;
/**
* The status of the evaluation run.
*/
status: string;
}
export declare namespace RunCancelResponse {
/**
* A ResponsesRunDataSource object describing a model sampling configuration.
*/
interface Responses {
/**
* Determines what populates the `item` namespace in this run's data source.
*/
source: Responses.FileContent | Responses.FileID | Responses.Responses;
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Used when sampling from a model. Dictates the structure of the messages passed
* into the model. Can either be a reference to a prebuilt trajectory (ie,
* `item.input_trajectory`), or a template with variable references to the `item`
* namespace.
*/
input_messages?: Responses.Template | Responses.ItemReference;
/**
* The name of the model to use for generating completions (e.g. "o3-mini").
*/
model?: string;
sampling_params?: Responses.SamplingParams;
}
namespace Responses {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
/**
* A EvalResponsesSource object describing a run data source configuration.
*/
interface Responses {
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Only include items created after this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_after?: number | null;
/**
* Only include items created before this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_before?: number | null;
/**
* Optional string to search the 'instructions' field. This is a query parameter
* used to select responses.
*/
instructions_search?: string | null;
/**
* Metadata filter for the responses. This is a query parameter used to select
* responses.
*/
metadata?: unknown | null;
/**
* The name of the model to find responses for. This is a query parameter used to
* select responses.
*/
model?: string | null;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* Sampling temperature. This is a query parameter used to select responses.
*/
temperature?: number | null;
/**
* List of tool names. This is a query parameter used to select responses.
*/
tools?: Array<string> | null;
/**
* Nucleus sampling parameter. This is a query parameter used to select responses.
*/
top_p?: number | null;
/**
* List of user identifiers. This is a query parameter used to select responses.
*/
users?: Array<string> | null;
}
interface Template {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
template: Array<Template.ChatMessage | Template.EvalItem>;
/**
* The type of input messages. Always `template`.
*/
type: 'template';
}
namespace Template {
interface ChatMessage {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role: string;
}
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
interface ItemReference {
/**
* A reference to a variable in the `item` namespace. Ie, "item.name"
*/
item_reference: string;
/**
* The type of input messages. Always `item_reference`.
*/
type: 'item_reference';
}
interface SamplingParams {
/**
* The maximum number of tokens in the generated output.
*/
max_completion_tokens?: number;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* A seed value to initialize the randomness, during sampling.
*/
seed?: number;
/**
* A higher temperature increases randomness in the outputs.
*/
temperature?: number;
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
text?: SamplingParams.Text;
/**
* An array of tools the model may call while generating a response. You can
* specify which tool to use by setting the `tool_choice` parameter.
*
* The two categories of tools you can provide the model are:
*
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
* capabilities, like
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
* Learn more about
* [built-in tools](https://platform.openai.com/docs/guides/tools).
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
* the model to call your own code. Learn more about
* [function calling](https://platform.openai.com/docs/guides/function-calling).
*/
tools?: Array<ResponsesAPI.Tool>;
/**
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
*/
top_p?: number;
}
namespace SamplingParams {
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
interface Text {
/**
* An object specifying the format that the model must output.
*
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
* ensures the model will match your supplied JSON schema. Learn more in the
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
*
* The default format is `{ "type": "text" }` with no additional options.
*
* **Not recommended for gpt-4o and newer models:**
*
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
* ensures the message the model generates is valid JSON. Using `json_schema` is
* preferred for models that support it.
*/
format?: ResponsesAPI.ResponseFormatTextConfig;
}
}
}
interface PerModelUsage {
/**
* The number of tokens retrieved from cache.
*/
cached_tokens: number;
/**
* The number of completion tokens generated.
*/
completion_tokens: number;
/**
* The number of invocations.
*/
invocation_count: number;
/**
* The name of the model.
*/
model_name: string;
/**
* The number of prompt tokens used.
*/
prompt_tokens: number;
/**
* The total number of tokens used.
*/
total_tokens: number;
}
interface PerTestingCriteriaResult {
/**
* Number of tests failed for this criteria.
*/
failed: number;
/**
* Number of tests passed for this criteria.
*/
passed: number;
/**
* A description of the testing criteria.
*/
testing_criteria: string;
}
/**
* Counters summarizing the outcomes of the evaluation run.
*/
interface ResultCounts {
/**
* Number of output items that resulted in an error.
*/
errored: number;
/**
* Number of output items that failed to pass the evaluation.
*/
failed: number;
/**
* Number of output items that passed the evaluation.
*/
passed: number;
/**
* Total number of executed output items.
*/
total: number;
}
}
export interface RunCreateParams {
/**
* Details about the run's data source.
*/
data_source: CreateEvalJSONLRunDataSource | CreateEvalCompletionsRunDataSource | RunCreateParams.CreateEvalResponsesRunDataSource;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
/**
* The name of the run.
*/
name?: string;
}
export declare namespace RunCreateParams {
/**
* A ResponsesRunDataSource object describing a model sampling configuration.
*/
interface CreateEvalResponsesRunDataSource {
/**
* Determines what populates the `item` namespace in this run's data source.
*/
source: CreateEvalResponsesRunDataSource.FileContent | CreateEvalResponsesRunDataSource.FileID | CreateEvalResponsesRunDataSource.Responses;
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Used when sampling from a model. Dictates the structure of the messages passed
* into the model. Can either be a reference to a prebuilt trajectory (ie,
* `item.input_trajectory`), or a template with variable references to the `item`
* namespace.
*/
input_messages?: CreateEvalResponsesRunDataSource.Template | CreateEvalResponsesRunDataSource.ItemReference;
/**
* The name of the model to use for generating completions (e.g. "o3-mini").
*/
model?: string;
sampling_params?: CreateEvalResponsesRunDataSource.SamplingParams;
}
namespace CreateEvalResponsesRunDataSource {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
/**
* A EvalResponsesSource object describing a run data source configuration.
*/
interface Responses {
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Only include items created after this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_after?: number | null;
/**
* Only include items created before this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_before?: number | null;
/**
* Optional string to search the 'instructions' field. This is a query parameter
* used to select responses.
*/
instructions_search?: string | null;
/**
* Metadata filter for the responses. This is a query parameter used to select
* responses.
*/
metadata?: unknown | null;
/**
* The name of the model to find responses for. This is a query parameter used to
* select responses.
*/
model?: string | null;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* Sampling temperature. This is a query parameter used to select responses.
*/
temperature?: number | null;
/**
* List of tool names. This is a query parameter used to select responses.
*/
tools?: Array<string> | null;
/**
* Nucleus sampling parameter. This is a query parameter used to select responses.
*/
top_p?: number | null;
/**
* List of user identifiers. This is a query parameter used to select responses.
*/
users?: Array<string> | null;
}
interface Template {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
template: Array<Template.ChatMessage | Template.EvalItem>;
/**
* The type of input messages. Always `template`.
*/
type: 'template';
}
namespace Template {
interface ChatMessage {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role: string;
}
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
interface ItemReference {
/**
* A reference to a variable in the `item` namespace. Ie, "item.name"
*/
item_reference: string;
/**
* The type of input messages. Always `item_reference`.
*/
type: 'item_reference';
}
interface SamplingParams {
/**
* The maximum number of tokens in the generated output.
*/
max_completion_tokens?: number;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* A seed value to initialize the randomness, during sampling.
*/
seed?: number;
/**
* A higher temperature increases randomness in the outputs.
*/
temperature?: number;
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
text?: SamplingParams.Text;
/**
* An array of tools the model may call while generating a response. You can
* specify which tool to use by setting the `tool_choice` parameter.
*
* The two categories of tools you can provide the model are:
*
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
* capabilities, like
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
* Learn more about
* [built-in tools](https://platform.openai.com/docs/guides/tools).
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
* the model to call your own code. Learn more about
* [function calling](https://platform.openai.com/docs/guides/function-calling).
*/
tools?: Array<ResponsesAPI.Tool>;
/**
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
*/
top_p?: number;
}
namespace SamplingParams {
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
interface Text {
/**
* An object specifying the format that the model must output.
*
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
* ensures the model will match your supplied JSON schema. Learn more in the
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
*
* The default format is `{ "type": "text" }` with no additional options.
*
* **Not recommended for gpt-4o and newer models:**
*
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
* ensures the message the model generates is valid JSON. Using `json_schema` is
* preferred for models that support it.
*/
format?: ResponsesAPI.ResponseFormatTextConfig;
}
}
}
}
export interface RunRetrieveParams {
/**
* The ID of the evaluation to retrieve runs for.
*/
eval_id: string;
}
export interface RunListParams extends CursorPageParams {
/**
* Sort order for runs by timestamp. Use `asc` for ascending order or `desc` for
* descending order. Defaults to `asc`.
*/
order?: 'asc' | 'desc';
/**
* Filter runs by status. One of `queued` | `in_progress` | `failed` | `completed`
* | `canceled`.
*/
status?: 'queued' | 'in_progress' | 'completed' | 'canceled' | 'failed';
}
export interface RunDeleteParams {
/**
* The ID of the evaluation to delete the run from.
*/
eval_id: string;
}
export interface RunCancelParams {
/**
* The ID of the evaluation whose run you want to cancel.
*/
eval_id: string;
}
export declare namespace Runs {
export { type CreateEvalCompletionsRunDataSource as CreateEvalCompletionsRunDataSource, type CreateEvalJSONLRunDataSource as CreateEvalJSONLRunDataSource, type EvalAPIError as EvalAPIError, type RunCreateResponse as RunCreateResponse, type RunRetrieveResponse as RunRetrieveResponse, type RunListResponse as RunListResponse, type RunDeleteResponse as RunDeleteResponse, type RunCancelResponse as RunCancelResponse, type RunListResponsesPage as RunListResponsesPage, type RunCreateParams as RunCreateParams, type RunRetrieveParams as RunRetrieveParams, type RunListParams as RunListParams, type RunDeleteParams as RunDeleteParams, type RunCancelParams as RunCancelParams, };
export { OutputItems as OutputItems, type OutputItemRetrieveResponse as OutputItemRetrieveResponse, type OutputItemListResponse as OutputItemListResponse, type OutputItemListResponsesPage as OutputItemListResponsesPage, type OutputItemRetrieveParams as OutputItemRetrieveParams, type OutputItemListParams as OutputItemListParams, };
}
//# sourceMappingURL=runs.d.mts.map
landvex/node_modules/openai/resources/evals/runs/runs.d.mts.map
Generated Vendored
+1
View File
File diff suppressed because one or more lines are too long
landvex/node_modules/openai/resources/evals/runs/runs.d.ts
Generated Vendored
+2388
View File
@@ -0,0 +1,2388 @@
import { APIResource } from "../../../core/resource.js";
import * as Shared from "../../shared.js";
import * as GraderModelsAPI from "../../graders/grader-models.js";
import * as ResponsesAPI from "../../responses/responses.js";
import * as CompletionsAPI from "../../chat/completions/completions.js";
import * as OutputItemsAPI from "./output-items.js";
import { OutputItemListParams, OutputItemListResponse, OutputItemListResponsesPage, OutputItemRetrieveParams, OutputItemRetrieveResponse, OutputItems } from "./output-items.js";
import { APIPromise } from "../../../core/api-promise.js";
import { CursorPage, type CursorPageParams, PagePromise } from "../../../core/pagination.js";
import { RequestOptions } from "../../../internal/request-options.js";
/**
* Manage and run evals in the OpenAI platform.
*/
export declare class Runs extends APIResource {
outputItems: OutputItemsAPI.OutputItems;
/**
* Kicks off a new run for a given evaluation, specifying the data source, and what
* model configuration to use to test. The datasource will be validated against the
* schema specified in the config of the evaluation.
*/
create(evalID: string, body: RunCreateParams, options?: RequestOptions): APIPromise<RunCreateResponse>;
/**
* Get an evaluation run by ID.
*/
retrieve(runID: string, params: RunRetrieveParams, options?: RequestOptions): APIPromise<RunRetrieveResponse>;
/**
* Get a list of runs for an evaluation.
*/
list(evalID: string, query?: RunListParams | null | undefined, options?: RequestOptions): PagePromise<RunListResponsesPage, RunListResponse>;
/**
* Delete an eval run.
*/
delete(runID: string, params: RunDeleteParams, options?: RequestOptions): APIPromise<RunDeleteResponse>;
/**
* Cancel an ongoing evaluation run.
*/
cancel(runID: string, params: RunCancelParams, options?: RequestOptions): APIPromise<RunCancelResponse>;
}
export type RunListResponsesPage = CursorPage<RunListResponse>;
/**
* A CompletionsRunDataSource object describing a model sampling configuration.
*/
export interface CreateEvalCompletionsRunDataSource {
/**
* Determines what populates the `item` namespace in this run's data source.
*/
source: CreateEvalCompletionsRunDataSource.FileContent | CreateEvalCompletionsRunDataSource.FileID | CreateEvalCompletionsRunDataSource.StoredCompletions;
/**
* The type of run data source. Always `completions`.
*/
type: 'completions';
/**
* Used when sampling from a model. Dictates the structure of the messages passed
* into the model. Can either be a reference to a prebuilt trajectory (ie,
* `item.input_trajectory`), or a template with variable references to the `item`
* namespace.
*/
input_messages?: CreateEvalCompletionsRunDataSource.Template | CreateEvalCompletionsRunDataSource.ItemReference;
/**
* The name of the model to use for generating completions (e.g. "o3-mini").
*/
model?: string;
sampling_params?: CreateEvalCompletionsRunDataSource.SamplingParams;
}
export declare namespace CreateEvalCompletionsRunDataSource {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
/**
* A StoredCompletionsRunDataSource configuration describing a set of filters
*/
interface StoredCompletions {
/**
* The type of source. Always `stored_completions`.
*/
type: 'stored_completions';
/**
* An optional Unix timestamp to filter items created after this time.
*/
created_after?: number | null;
/**
* An optional Unix timestamp to filter items created before this time.
*/
created_before?: number | null;
/**
* An optional maximum number of items to return.
*/
limit?: number | null;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
/**
* An optional model to filter by (e.g., 'gpt-4o').
*/
model?: string | null;
}
interface Template {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
template: Array<ResponsesAPI.EasyInputMessage | Template.EvalItem>;
/**
* The type of input messages. Always `template`.
*/
type: 'template';
}
namespace Template {
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
interface ItemReference {
/**
* A reference to a variable in the `item` namespace. Ie, "item.input_trajectory"
*/
item_reference: string;
/**
* The type of input messages. Always `item_reference`.
*/
type: 'item_reference';
}
interface SamplingParams {
/**
* The maximum number of tokens in the generated output.
*/
max_completion_tokens?: number;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* An object specifying the format that the model must output.
*
* Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured
* Outputs which ensures the model will match your supplied JSON schema. Learn more
* in the
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
*
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
* ensures the message the model generates is valid JSON. Using `json_schema` is
* preferred for models that support it.
*/
response_format?: Shared.ResponseFormatText | Shared.ResponseFormatJSONSchema | Shared.ResponseFormatJSONObject;
/**
* A seed value to initialize the randomness, during sampling.
*/
seed?: number;
/**
* A higher temperature increases randomness in the outputs.
*/
temperature?: number;
/**
* A list of tools the model may call. Currently, only functions are supported as a
* tool. Use this to provide a list of functions the model may generate JSON inputs
* for. A max of 128 functions are supported.
*/
tools?: Array<CompletionsAPI.ChatCompletionFunctionTool>;
/**
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
*/
top_p?: number;
}
}
/**
* A JsonlRunDataSource object with that specifies a JSONL file that matches the
* eval
*/
export interface CreateEvalJSONLRunDataSource {
/**
* Determines what populates the `item` namespace in the data source.
*/
source: CreateEvalJSONLRunDataSource.FileContent | CreateEvalJSONLRunDataSource.FileID;
/**
* The type of data source. Always `jsonl`.
*/
type: 'jsonl';
}
export declare namespace CreateEvalJSONLRunDataSource {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
}
/**
* An object representing an error response from the Eval API.
*/
export interface EvalAPIError {
/**
* The error code.
*/
code: string;
/**
* The error message.
*/
message: string;
}
/**
* A schema representing an evaluation run.
*/
export interface RunCreateResponse {
/**
* Unique identifier for the evaluation run.
*/
id: string;
/**
* Unix timestamp (in seconds) when the evaluation run was created.
*/
created_at: number;
/**
* Information about the run's data source.
*/
data_source: CreateEvalJSONLRunDataSource | CreateEvalCompletionsRunDataSource | RunCreateResponse.Responses;
/**
* An object representing an error response from the Eval API.
*/
error: EvalAPIError;
/**
* The identifier of the associated evaluation.
*/
eval_id: string;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The model that is evaluated, if applicable.
*/
model: string;
/**
* The name of the evaluation run.
*/
name: string;
/**
* The type of the object. Always "eval.run".
*/
object: 'eval.run';
/**
* Usage statistics for each model during the evaluation run.
*/
per_model_usage: Array<RunCreateResponse.PerModelUsage>;
/**
* Results per testing criteria applied during the evaluation run.
*/
per_testing_criteria_results: Array<RunCreateResponse.PerTestingCriteriaResult>;
/**
* The URL to the rendered evaluation run report on the UI dashboard.
*/
report_url: string;
/**
* Counters summarizing the outcomes of the evaluation run.
*/
result_counts: RunCreateResponse.ResultCounts;
/**
* The status of the evaluation run.
*/
status: string;
}
export declare namespace RunCreateResponse {
/**
* A ResponsesRunDataSource object describing a model sampling configuration.
*/
interface Responses {
/**
* Determines what populates the `item` namespace in this run's data source.
*/
source: Responses.FileContent | Responses.FileID | Responses.Responses;
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Used when sampling from a model. Dictates the structure of the messages passed
* into the model. Can either be a reference to a prebuilt trajectory (ie,
* `item.input_trajectory`), or a template with variable references to the `item`
* namespace.
*/
input_messages?: Responses.Template | Responses.ItemReference;
/**
* The name of the model to use for generating completions (e.g. "o3-mini").
*/
model?: string;
sampling_params?: Responses.SamplingParams;
}
namespace Responses {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
/**
* A EvalResponsesSource object describing a run data source configuration.
*/
interface Responses {
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Only include items created after this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_after?: number | null;
/**
* Only include items created before this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_before?: number | null;
/**
* Optional string to search the 'instructions' field. This is a query parameter
* used to select responses.
*/
instructions_search?: string | null;
/**
* Metadata filter for the responses. This is a query parameter used to select
* responses.
*/
metadata?: unknown | null;
/**
* The name of the model to find responses for. This is a query parameter used to
* select responses.
*/
model?: string | null;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* Sampling temperature. This is a query parameter used to select responses.
*/
temperature?: number | null;
/**
* List of tool names. This is a query parameter used to select responses.
*/
tools?: Array<string> | null;
/**
* Nucleus sampling parameter. This is a query parameter used to select responses.
*/
top_p?: number | null;
/**
* List of user identifiers. This is a query parameter used to select responses.
*/
users?: Array<string> | null;
}
interface Template {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
template: Array<Template.ChatMessage | Template.EvalItem>;
/**
* The type of input messages. Always `template`.
*/
type: 'template';
}
namespace Template {
interface ChatMessage {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role: string;
}
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
interface ItemReference {
/**
* A reference to a variable in the `item` namespace. Ie, "item.name"
*/
item_reference: string;
/**
* The type of input messages. Always `item_reference`.
*/
type: 'item_reference';
}
interface SamplingParams {
/**
* The maximum number of tokens in the generated output.
*/
max_completion_tokens?: number;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* A seed value to initialize the randomness, during sampling.
*/
seed?: number;
/**
* A higher temperature increases randomness in the outputs.
*/
temperature?: number;
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
text?: SamplingParams.Text;
/**
* An array of tools the model may call while generating a response. You can
* specify which tool to use by setting the `tool_choice` parameter.
*
* The two categories of tools you can provide the model are:
*
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
* capabilities, like
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
* Learn more about
* [built-in tools](https://platform.openai.com/docs/guides/tools).
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
* the model to call your own code. Learn more about
* [function calling](https://platform.openai.com/docs/guides/function-calling).
*/
tools?: Array<ResponsesAPI.Tool>;
/**
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
*/
top_p?: number;
}
namespace SamplingParams {
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
interface Text {
/**
* An object specifying the format that the model must output.
*
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
* ensures the model will match your supplied JSON schema. Learn more in the
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
*
* The default format is `{ "type": "text" }` with no additional options.
*
* **Not recommended for gpt-4o and newer models:**
*
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
* ensures the message the model generates is valid JSON. Using `json_schema` is
* preferred for models that support it.
*/
format?: ResponsesAPI.ResponseFormatTextConfig;
}
}
}
interface PerModelUsage {
/**
* The number of tokens retrieved from cache.
*/
cached_tokens: number;
/**
* The number of completion tokens generated.
*/
completion_tokens: number;
/**
* The number of invocations.
*/
invocation_count: number;
/**
* The name of the model.
*/
model_name: string;
/**
* The number of prompt tokens used.
*/
prompt_tokens: number;
/**
* The total number of tokens used.
*/
total_tokens: number;
}
interface PerTestingCriteriaResult {
/**
* Number of tests failed for this criteria.
*/
failed: number;
/**
* Number of tests passed for this criteria.
*/
passed: number;
/**
* A description of the testing criteria.
*/
testing_criteria: string;
}
/**
* Counters summarizing the outcomes of the evaluation run.
*/
interface ResultCounts {
/**
* Number of output items that resulted in an error.
*/
errored: number;
/**
* Number of output items that failed to pass the evaluation.
*/
failed: number;
/**
* Number of output items that passed the evaluation.
*/
passed: number;
/**
* Total number of executed output items.
*/
total: number;
}
}
/**
* A schema representing an evaluation run.
*/
export interface RunRetrieveResponse {
/**
* Unique identifier for the evaluation run.
*/
id: string;
/**
* Unix timestamp (in seconds) when the evaluation run was created.
*/
created_at: number;
/**
* Information about the run's data source.
*/
data_source: CreateEvalJSONLRunDataSource | CreateEvalCompletionsRunDataSource | RunRetrieveResponse.Responses;
/**
* An object representing an error response from the Eval API.
*/
error: EvalAPIError;
/**
* The identifier of the associated evaluation.
*/
eval_id: string;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The model that is evaluated, if applicable.
*/
model: string;
/**
* The name of the evaluation run.
*/
name: string;
/**
* The type of the object. Always "eval.run".
*/
object: 'eval.run';
/**
* Usage statistics for each model during the evaluation run.
*/
per_model_usage: Array<RunRetrieveResponse.PerModelUsage>;
/**
* Results per testing criteria applied during the evaluation run.
*/
per_testing_criteria_results: Array<RunRetrieveResponse.PerTestingCriteriaResult>;
/**
* The URL to the rendered evaluation run report on the UI dashboard.
*/
report_url: string;
/**
* Counters summarizing the outcomes of the evaluation run.
*/
result_counts: RunRetrieveResponse.ResultCounts;
/**
* The status of the evaluation run.
*/
status: string;
}
export declare namespace RunRetrieveResponse {
/**
* A ResponsesRunDataSource object describing a model sampling configuration.
*/
interface Responses {
/**
* Determines what populates the `item` namespace in this run's data source.
*/
source: Responses.FileContent | Responses.FileID | Responses.Responses;
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Used when sampling from a model. Dictates the structure of the messages passed
* into the model. Can either be a reference to a prebuilt trajectory (ie,
* `item.input_trajectory`), or a template with variable references to the `item`
* namespace.
*/
input_messages?: Responses.Template | Responses.ItemReference;
/**
* The name of the model to use for generating completions (e.g. "o3-mini").
*/
model?: string;
sampling_params?: Responses.SamplingParams;
}
namespace Responses {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
/**
* A EvalResponsesSource object describing a run data source configuration.
*/
interface Responses {
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Only include items created after this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_after?: number | null;
/**
* Only include items created before this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_before?: number | null;
/**
* Optional string to search the 'instructions' field. This is a query parameter
* used to select responses.
*/
instructions_search?: string | null;
/**
* Metadata filter for the responses. This is a query parameter used to select
* responses.
*/
metadata?: unknown | null;
/**
* The name of the model to find responses for. This is a query parameter used to
* select responses.
*/
model?: string | null;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* Sampling temperature. This is a query parameter used to select responses.
*/
temperature?: number | null;
/**
* List of tool names. This is a query parameter used to select responses.
*/
tools?: Array<string> | null;
/**
* Nucleus sampling parameter. This is a query parameter used to select responses.
*/
top_p?: number | null;
/**
* List of user identifiers. This is a query parameter used to select responses.
*/
users?: Array<string> | null;
}
interface Template {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
template: Array<Template.ChatMessage | Template.EvalItem>;
/**
* The type of input messages. Always `template`.
*/
type: 'template';
}
namespace Template {
interface ChatMessage {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role: string;
}
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
interface ItemReference {
/**
* A reference to a variable in the `item` namespace. Ie, "item.name"
*/
item_reference: string;
/**
* The type of input messages. Always `item_reference`.
*/
type: 'item_reference';
}
interface SamplingParams {
/**
* The maximum number of tokens in the generated output.
*/
max_completion_tokens?: number;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* A seed value to initialize the randomness, during sampling.
*/
seed?: number;
/**
* A higher temperature increases randomness in the outputs.
*/
temperature?: number;
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
text?: SamplingParams.Text;
/**
* An array of tools the model may call while generating a response. You can
* specify which tool to use by setting the `tool_choice` parameter.
*
* The two categories of tools you can provide the model are:
*
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
* capabilities, like
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
* Learn more about
* [built-in tools](https://platform.openai.com/docs/guides/tools).
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
* the model to call your own code. Learn more about
* [function calling](https://platform.openai.com/docs/guides/function-calling).
*/
tools?: Array<ResponsesAPI.Tool>;
/**
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
*/
top_p?: number;
}
namespace SamplingParams {
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
interface Text {
/**
* An object specifying the format that the model must output.
*
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
* ensures the model will match your supplied JSON schema. Learn more in the
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
*
* The default format is `{ "type": "text" }` with no additional options.
*
* **Not recommended for gpt-4o and newer models:**
*
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
* ensures the message the model generates is valid JSON. Using `json_schema` is
* preferred for models that support it.
*/
format?: ResponsesAPI.ResponseFormatTextConfig;
}
}
}
interface PerModelUsage {
/**
* The number of tokens retrieved from cache.
*/
cached_tokens: number;
/**
* The number of completion tokens generated.
*/
completion_tokens: number;
/**
* The number of invocations.
*/
invocation_count: number;
/**
* The name of the model.
*/
model_name: string;
/**
* The number of prompt tokens used.
*/
prompt_tokens: number;
/**
* The total number of tokens used.
*/
total_tokens: number;
}
interface PerTestingCriteriaResult {
/**
* Number of tests failed for this criteria.
*/
failed: number;
/**
* Number of tests passed for this criteria.
*/
passed: number;
/**
* A description of the testing criteria.
*/
testing_criteria: string;
}
/**
* Counters summarizing the outcomes of the evaluation run.
*/
interface ResultCounts {
/**
* Number of output items that resulted in an error.
*/
errored: number;
/**
* Number of output items that failed to pass the evaluation.
*/
failed: number;
/**
* Number of output items that passed the evaluation.
*/
passed: number;
/**
* Total number of executed output items.
*/
total: number;
}
}
/**
* A schema representing an evaluation run.
*/
export interface RunListResponse {
/**
* Unique identifier for the evaluation run.
*/
id: string;
/**
* Unix timestamp (in seconds) when the evaluation run was created.
*/
created_at: number;
/**
* Information about the run's data source.
*/
data_source: CreateEvalJSONLRunDataSource | CreateEvalCompletionsRunDataSource | RunListResponse.Responses;
/**
* An object representing an error response from the Eval API.
*/
error: EvalAPIError;
/**
* The identifier of the associated evaluation.
*/
eval_id: string;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The model that is evaluated, if applicable.
*/
model: string;
/**
* The name of the evaluation run.
*/
name: string;
/**
* The type of the object. Always "eval.run".
*/
object: 'eval.run';
/**
* Usage statistics for each model during the evaluation run.
*/
per_model_usage: Array<RunListResponse.PerModelUsage>;
/**
* Results per testing criteria applied during the evaluation run.
*/
per_testing_criteria_results: Array<RunListResponse.PerTestingCriteriaResult>;
/**
* The URL to the rendered evaluation run report on the UI dashboard.
*/
report_url: string;
/**
* Counters summarizing the outcomes of the evaluation run.
*/
result_counts: RunListResponse.ResultCounts;
/**
* The status of the evaluation run.
*/
status: string;
}
export declare namespace RunListResponse {
/**
* A ResponsesRunDataSource object describing a model sampling configuration.
*/
interface Responses {
/**
* Determines what populates the `item` namespace in this run's data source.
*/
source: Responses.FileContent | Responses.FileID | Responses.Responses;
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Used when sampling from a model. Dictates the structure of the messages passed
* into the model. Can either be a reference to a prebuilt trajectory (ie,
* `item.input_trajectory`), or a template with variable references to the `item`
* namespace.
*/
input_messages?: Responses.Template | Responses.ItemReference;
/**
* The name of the model to use for generating completions (e.g. "o3-mini").
*/
model?: string;
sampling_params?: Responses.SamplingParams;
}
namespace Responses {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
/**
* A EvalResponsesSource object describing a run data source configuration.
*/
interface Responses {
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Only include items created after this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_after?: number | null;
/**
* Only include items created before this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_before?: number | null;
/**
* Optional string to search the 'instructions' field. This is a query parameter
* used to select responses.
*/
instructions_search?: string | null;
/**
* Metadata filter for the responses. This is a query parameter used to select
* responses.
*/
metadata?: unknown | null;
/**
* The name of the model to find responses for. This is a query parameter used to
* select responses.
*/
model?: string | null;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* Sampling temperature. This is a query parameter used to select responses.
*/
temperature?: number | null;
/**
* List of tool names. This is a query parameter used to select responses.
*/
tools?: Array<string> | null;
/**
* Nucleus sampling parameter. This is a query parameter used to select responses.
*/
top_p?: number | null;
/**
* List of user identifiers. This is a query parameter used to select responses.
*/
users?: Array<string> | null;
}
interface Template {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
template: Array<Template.ChatMessage | Template.EvalItem>;
/**
* The type of input messages. Always `template`.
*/
type: 'template';
}
namespace Template {
interface ChatMessage {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role: string;
}
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
interface ItemReference {
/**
* A reference to a variable in the `item` namespace. Ie, "item.name"
*/
item_reference: string;
/**
* The type of input messages. Always `item_reference`.
*/
type: 'item_reference';
}
interface SamplingParams {
/**
* The maximum number of tokens in the generated output.
*/
max_completion_tokens?: number;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* A seed value to initialize the randomness, during sampling.
*/
seed?: number;
/**
* A higher temperature increases randomness in the outputs.
*/
temperature?: number;
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
text?: SamplingParams.Text;
/**
* An array of tools the model may call while generating a response. You can
* specify which tool to use by setting the `tool_choice` parameter.
*
* The two categories of tools you can provide the model are:
*
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
* capabilities, like
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
* Learn more about
* [built-in tools](https://platform.openai.com/docs/guides/tools).
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
* the model to call your own code. Learn more about
* [function calling](https://platform.openai.com/docs/guides/function-calling).
*/
tools?: Array<ResponsesAPI.Tool>;
/**
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
*/
top_p?: number;
}
namespace SamplingParams {
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
interface Text {
/**
* An object specifying the format that the model must output.
*
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
* ensures the model will match your supplied JSON schema. Learn more in the
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
*
* The default format is `{ "type": "text" }` with no additional options.
*
* **Not recommended for gpt-4o and newer models:**
*
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
* ensures the message the model generates is valid JSON. Using `json_schema` is
* preferred for models that support it.
*/
format?: ResponsesAPI.ResponseFormatTextConfig;
}
}
}
interface PerModelUsage {
/**
* The number of tokens retrieved from cache.
*/
cached_tokens: number;
/**
* The number of completion tokens generated.
*/
completion_tokens: number;
/**
* The number of invocations.
*/
invocation_count: number;
/**
* The name of the model.
*/
model_name: string;
/**
* The number of prompt tokens used.
*/
prompt_tokens: number;
/**
* The total number of tokens used.
*/
total_tokens: number;
}
interface PerTestingCriteriaResult {
/**
* Number of tests failed for this criteria.
*/
failed: number;
/**
* Number of tests passed for this criteria.
*/
passed: number;
/**
* A description of the testing criteria.
*/
testing_criteria: string;
}
/**
* Counters summarizing the outcomes of the evaluation run.
*/
interface ResultCounts {
/**
* Number of output items that resulted in an error.
*/
errored: number;
/**
* Number of output items that failed to pass the evaluation.
*/
failed: number;
/**
* Number of output items that passed the evaluation.
*/
passed: number;
/**
* Total number of executed output items.
*/
total: number;
}
}
export interface RunDeleteResponse {
deleted?: boolean;
object?: string;
run_id?: string;
}
/**
* A schema representing an evaluation run.
*/
export interface RunCancelResponse {
/**
* Unique identifier for the evaluation run.
*/
id: string;
/**
* Unix timestamp (in seconds) when the evaluation run was created.
*/
created_at: number;
/**
* Information about the run's data source.
*/
data_source: CreateEvalJSONLRunDataSource | CreateEvalCompletionsRunDataSource | RunCancelResponse.Responses;
/**
* An object representing an error response from the Eval API.
*/
error: EvalAPIError;
/**
* The identifier of the associated evaluation.
*/
eval_id: string;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata: Shared.Metadata | null;
/**
* The model that is evaluated, if applicable.
*/
model: string;
/**
* The name of the evaluation run.
*/
name: string;
/**
* The type of the object. Always "eval.run".
*/
object: 'eval.run';
/**
* Usage statistics for each model during the evaluation run.
*/
per_model_usage: Array<RunCancelResponse.PerModelUsage>;
/**
* Results per testing criteria applied during the evaluation run.
*/
per_testing_criteria_results: Array<RunCancelResponse.PerTestingCriteriaResult>;
/**
* The URL to the rendered evaluation run report on the UI dashboard.
*/
report_url: string;
/**
* Counters summarizing the outcomes of the evaluation run.
*/
result_counts: RunCancelResponse.ResultCounts;
/**
* The status of the evaluation run.
*/
status: string;
}
export declare namespace RunCancelResponse {
/**
* A ResponsesRunDataSource object describing a model sampling configuration.
*/
interface Responses {
/**
* Determines what populates the `item` namespace in this run's data source.
*/
source: Responses.FileContent | Responses.FileID | Responses.Responses;
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Used when sampling from a model. Dictates the structure of the messages passed
* into the model. Can either be a reference to a prebuilt trajectory (ie,
* `item.input_trajectory`), or a template with variable references to the `item`
* namespace.
*/
input_messages?: Responses.Template | Responses.ItemReference;
/**
* The name of the model to use for generating completions (e.g. "o3-mini").
*/
model?: string;
sampling_params?: Responses.SamplingParams;
}
namespace Responses {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
/**
* A EvalResponsesSource object describing a run data source configuration.
*/
interface Responses {
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Only include items created after this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_after?: number | null;
/**
* Only include items created before this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_before?: number | null;
/**
* Optional string to search the 'instructions' field. This is a query parameter
* used to select responses.
*/
instructions_search?: string | null;
/**
* Metadata filter for the responses. This is a query parameter used to select
* responses.
*/
metadata?: unknown | null;
/**
* The name of the model to find responses for. This is a query parameter used to
* select responses.
*/
model?: string | null;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* Sampling temperature. This is a query parameter used to select responses.
*/
temperature?: number | null;
/**
* List of tool names. This is a query parameter used to select responses.
*/
tools?: Array<string> | null;
/**
* Nucleus sampling parameter. This is a query parameter used to select responses.
*/
top_p?: number | null;
/**
* List of user identifiers. This is a query parameter used to select responses.
*/
users?: Array<string> | null;
}
interface Template {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
template: Array<Template.ChatMessage | Template.EvalItem>;
/**
* The type of input messages. Always `template`.
*/
type: 'template';
}
namespace Template {
interface ChatMessage {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role: string;
}
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
interface ItemReference {
/**
* A reference to a variable in the `item` namespace. Ie, "item.name"
*/
item_reference: string;
/**
* The type of input messages. Always `item_reference`.
*/
type: 'item_reference';
}
interface SamplingParams {
/**
* The maximum number of tokens in the generated output.
*/
max_completion_tokens?: number;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* A seed value to initialize the randomness, during sampling.
*/
seed?: number;
/**
* A higher temperature increases randomness in the outputs.
*/
temperature?: number;
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
text?: SamplingParams.Text;
/**
* An array of tools the model may call while generating a response. You can
* specify which tool to use by setting the `tool_choice` parameter.
*
* The two categories of tools you can provide the model are:
*
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
* capabilities, like
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
* Learn more about
* [built-in tools](https://platform.openai.com/docs/guides/tools).
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
* the model to call your own code. Learn more about
* [function calling](https://platform.openai.com/docs/guides/function-calling).
*/
tools?: Array<ResponsesAPI.Tool>;
/**
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
*/
top_p?: number;
}
namespace SamplingParams {
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
interface Text {
/**
* An object specifying the format that the model must output.
*
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
* ensures the model will match your supplied JSON schema. Learn more in the
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
*
* The default format is `{ "type": "text" }` with no additional options.
*
* **Not recommended for gpt-4o and newer models:**
*
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
* ensures the message the model generates is valid JSON. Using `json_schema` is
* preferred for models that support it.
*/
format?: ResponsesAPI.ResponseFormatTextConfig;
}
}
}
interface PerModelUsage {
/**
* The number of tokens retrieved from cache.
*/
cached_tokens: number;
/**
* The number of completion tokens generated.
*/
completion_tokens: number;
/**
* The number of invocations.
*/
invocation_count: number;
/**
* The name of the model.
*/
model_name: string;
/**
* The number of prompt tokens used.
*/
prompt_tokens: number;
/**
* The total number of tokens used.
*/
total_tokens: number;
}
interface PerTestingCriteriaResult {
/**
* Number of tests failed for this criteria.
*/
failed: number;
/**
* Number of tests passed for this criteria.
*/
passed: number;
/**
* A description of the testing criteria.
*/
testing_criteria: string;
}
/**
* Counters summarizing the outcomes of the evaluation run.
*/
interface ResultCounts {
/**
* Number of output items that resulted in an error.
*/
errored: number;
/**
* Number of output items that failed to pass the evaluation.
*/
failed: number;
/**
* Number of output items that passed the evaluation.
*/
passed: number;
/**
* Total number of executed output items.
*/
total: number;
}
}
export interface RunCreateParams {
/**
* Details about the run's data source.
*/
data_source: CreateEvalJSONLRunDataSource | CreateEvalCompletionsRunDataSource | RunCreateParams.CreateEvalResponsesRunDataSource;
/**
* Set of 16 key-value pairs that can be attached to an object. This can be useful
* for storing additional information about the object in a structured format, and
* querying for objects via API or the dashboard.
*
* Keys are strings with a maximum length of 64 characters. Values are strings with
* a maximum length of 512 characters.
*/
metadata?: Shared.Metadata | null;
/**
* The name of the run.
*/
name?: string;
}
export declare namespace RunCreateParams {
/**
* A ResponsesRunDataSource object describing a model sampling configuration.
*/
interface CreateEvalResponsesRunDataSource {
/**
* Determines what populates the `item` namespace in this run's data source.
*/
source: CreateEvalResponsesRunDataSource.FileContent | CreateEvalResponsesRunDataSource.FileID | CreateEvalResponsesRunDataSource.Responses;
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Used when sampling from a model. Dictates the structure of the messages passed
* into the model. Can either be a reference to a prebuilt trajectory (ie,
* `item.input_trajectory`), or a template with variable references to the `item`
* namespace.
*/
input_messages?: CreateEvalResponsesRunDataSource.Template | CreateEvalResponsesRunDataSource.ItemReference;
/**
* The name of the model to use for generating completions (e.g. "o3-mini").
*/
model?: string;
sampling_params?: CreateEvalResponsesRunDataSource.SamplingParams;
}
namespace CreateEvalResponsesRunDataSource {
interface FileContent {
/**
* The content of the jsonl file.
*/
content: Array<FileContent.Content>;
/**
* The type of jsonl source. Always `file_content`.
*/
type: 'file_content';
}
namespace FileContent {
interface Content {
item: {
[key: string]: unknown;
};
sample?: {
[key: string]: unknown;
};
}
}
interface FileID {
/**
* The identifier of the file.
*/
id: string;
/**
* The type of jsonl source. Always `file_id`.
*/
type: 'file_id';
}
/**
* A EvalResponsesSource object describing a run data source configuration.
*/
interface Responses {
/**
* The type of run data source. Always `responses`.
*/
type: 'responses';
/**
* Only include items created after this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_after?: number | null;
/**
* Only include items created before this timestamp (inclusive). This is a query
* parameter used to select responses.
*/
created_before?: number | null;
/**
* Optional string to search the 'instructions' field. This is a query parameter
* used to select responses.
*/
instructions_search?: string | null;
/**
* Metadata filter for the responses. This is a query parameter used to select
* responses.
*/
metadata?: unknown | null;
/**
* The name of the model to find responses for. This is a query parameter used to
* select responses.
*/
model?: string | null;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* Sampling temperature. This is a query parameter used to select responses.
*/
temperature?: number | null;
/**
* List of tool names. This is a query parameter used to select responses.
*/
tools?: Array<string> | null;
/**
* Nucleus sampling parameter. This is a query parameter used to select responses.
*/
top_p?: number | null;
/**
* List of user identifiers. This is a query parameter used to select responses.
*/
users?: Array<string> | null;
}
interface Template {
/**
* A list of chat messages forming the prompt or context. May include variable
* references to the `item` namespace, ie {{item.name}}.
*/
template: Array<Template.ChatMessage | Template.EvalItem>;
/**
* The type of input messages. Always `template`.
*/
type: 'template';
}
namespace Template {
interface ChatMessage {
/**
* The content of the message.
*/
content: string;
/**
* The role of the message (e.g. "system", "assistant", "user").
*/
role: string;
}
/**
* A message input to the model with a role indicating instruction following
* hierarchy. Instructions given with the `developer` or `system` role take
* precedence over instructions given with the `user` role. Messages with the
* `assistant` role are presumed to have been generated by the model in previous
* interactions.
*/
interface EvalItem {
/**
* Inputs to the model - can contain template strings. Supports text, output text,
* input images, and input audio, either as a single item or an array of items.
*/
content: string | ResponsesAPI.ResponseInputText | EvalItem.OutputText | EvalItem.InputImage | ResponsesAPI.ResponseInputAudio | GraderModelsAPI.GraderInputs;
/**
* The role of the message input. One of `user`, `assistant`, `system`, or
* `developer`.
*/
role: 'user' | 'assistant' | 'system' | 'developer';
/**
* The type of the message input. Always `message`.
*/
type?: 'message';
}
namespace EvalItem {
/**
* A text output from the model.
*/
interface OutputText {
/**
* The text output from the model.
*/
text: string;
/**
* The type of the output text. Always `output_text`.
*/
type: 'output_text';
}
/**
* An image input block used within EvalItem content arrays.
*/
interface InputImage {
/**
* The URL of the image input.
*/
image_url: string;
/**
* The type of the image input. Always `input_image`.
*/
type: 'input_image';
/**
* The detail level of the image to be sent to the model. One of `high`, `low`, or
* `auto`. Defaults to `auto`.
*/
detail?: string;
}
}
}
interface ItemReference {
/**
* A reference to a variable in the `item` namespace. Ie, "item.name"
*/
item_reference: string;
/**
* The type of input messages. Always `item_reference`.
*/
type: 'item_reference';
}
interface SamplingParams {
/**
* The maximum number of tokens in the generated output.
*/
max_completion_tokens?: number;
/**
* Constrains effort on reasoning for
* [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
* supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
* Reducing reasoning effort can result in faster responses and fewer tokens used
* on reasoning in a response.
*
* - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
* reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
* calls are supported for all reasoning values in gpt-5.1.
* - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
* support `none`.
* - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
* - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
*/
reasoning_effort?: Shared.ReasoningEffort | null;
/**
* A seed value to initialize the randomness, during sampling.
*/
seed?: number;
/**
* A higher temperature increases randomness in the outputs.
*/
temperature?: number;
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
text?: SamplingParams.Text;
/**
* An array of tools the model may call while generating a response. You can
* specify which tool to use by setting the `tool_choice` parameter.
*
* The two categories of tools you can provide the model are:
*
* - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
* capabilities, like
* [web search](https://platform.openai.com/docs/guides/tools-web-search) or
* [file search](https://platform.openai.com/docs/guides/tools-file-search).
* Learn more about
* [built-in tools](https://platform.openai.com/docs/guides/tools).
* - **Function calls (custom tools)**: Functions that are defined by you, enabling
* the model to call your own code. Learn more about
* [function calling](https://platform.openai.com/docs/guides/function-calling).
*/
tools?: Array<ResponsesAPI.Tool>;
/**
* An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
*/
top_p?: number;
}
namespace SamplingParams {
/**
* Configuration options for a text response from the model. Can be plain text or
* structured JSON data. Learn more:
*
* - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
* - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
*/
interface Text {
/**
* An object specifying the format that the model must output.
*
* Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
* ensures the model will match your supplied JSON schema. Learn more in the
* [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
*
* The default format is `{ "type": "text" }` with no additional options.
*
* **Not recommended for gpt-4o and newer models:**
*
* Setting to `{ "type": "json_object" }` enables the older JSON mode, which
* ensures the message the model generates is valid JSON. Using `json_schema` is
* preferred for models that support it.
*/
format?: ResponsesAPI.ResponseFormatTextConfig;
}
}
}
}
export interface RunRetrieveParams {
/**
* The ID of the evaluation to retrieve runs for.
*/
eval_id: string;
}
export interface RunListParams extends CursorPageParams {
/**
* Sort order for runs by timestamp. Use `asc` for ascending order or `desc` for
* descending order. Defaults to `asc`.
*/
order?: 'asc' | 'desc';
/**
* Filter runs by status. One of `queued` | `in_progress` | `failed` | `completed`
* | `canceled`.
*/
status?: 'queued' | 'in_progress' | 'completed' | 'canceled' | 'failed';
}
export interface RunDeleteParams {
/**
* The ID of the evaluation to delete the run from.
*/
eval_id: string;
}
export interface RunCancelParams {
/**
* The ID of the evaluation whose run you want to cancel.
*/
eval_id: string;
}
export declare namespace Runs {
export { type CreateEvalCompletionsRunDataSource as CreateEvalCompletionsRunDataSource, type CreateEvalJSONLRunDataSource as CreateEvalJSONLRunDataSource, type EvalAPIError as EvalAPIError, type RunCreateResponse as RunCreateResponse, type RunRetrieveResponse as RunRetrieveResponse, type RunListResponse as RunListResponse, type RunDeleteResponse as RunDeleteResponse, type RunCancelResponse as RunCancelResponse, type RunListResponsesPage as RunListResponsesPage, type RunCreateParams as RunCreateParams, type RunRetrieveParams as RunRetrieveParams, type RunListParams as RunListParams, type RunDeleteParams as RunDeleteParams, type RunCancelParams as RunCancelParams, };
export { OutputItems as OutputItems, type OutputItemRetrieveResponse as OutputItemRetrieveResponse, type OutputItemListResponse as OutputItemListResponse, type OutputItemListResponsesPage as OutputItemListResponsesPage, type OutputItemRetrieveParams as OutputItemRetrieveParams, type OutputItemListParams as OutputItemListParams, };
}
//# sourceMappingURL=runs.d.ts.map
landvex/node_modules/openai/resources/evals/runs/runs.d.ts.map
Generated Vendored
+1
View File
File diff suppressed because one or more lines are too long
landvex/node_modules/openai/resources/evals/runs/runs.js
Generated Vendored
+74
View File
@@ -0,0 +1,74 @@
"use strict";
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
Object.defineProperty(exports, "__esModule", { value: true });
exports.Runs = void 0;
const tslib_1 = require("../../../internal/tslib.js");
const resource_1 = require("../../../core/resource.js");
const OutputItemsAPI = tslib_1.__importStar(require("./output-items.js"));
const output_items_1 = require("./output-items.js");
const pagination_1 = require("../../../core/pagination.js");
const path_1 = require("../../../internal/utils/path.js");
/**
* Manage and run evals in the OpenAI platform.
*/
class Runs extends resource_1.APIResource {
constructor() {
super(...arguments);
this.outputItems = new OutputItemsAPI.OutputItems(this._client);
}
/**
* Kicks off a new run for a given evaluation, specifying the data source, and what
* model configuration to use to test. The datasource will be validated against the
* schema specified in the config of the evaluation.
*/
create(evalID, body, options) {
return this._client.post((0, path_1.path) `/evals/${evalID}/runs`, {
body,
...options,
__security: { bearerAuth: true },
});
}
/**
* Get an evaluation run by ID.
*/
retrieve(runID, params, options) {
const { eval_id } = params;
return this._client.get((0, path_1.path) `/evals/${eval_id}/runs/${runID}`, {
...options,
__security: { bearerAuth: true },
});
}
/**
* Get a list of runs for an evaluation.
*/
list(evalID, query = {}, options) {
return this._client.getAPIList((0, path_1.path) `/evals/${evalID}/runs`, (pagination_1.CursorPage), {
query,
...options,
__security: { bearerAuth: true },
});
}
/**
* Delete an eval run.
*/
delete(runID, params, options) {
const { eval_id } = params;
return this._client.delete((0, path_1.path) `/evals/${eval_id}/runs/${runID}`, {
...options,
__security: { bearerAuth: true },
});
}
/**
* Cancel an ongoing evaluation run.
*/
cancel(runID, params, options) {
const { eval_id } = params;
return this._client.post((0, path_1.path) `/evals/${eval_id}/runs/${runID}`, {
...options,
__security: { bearerAuth: true },
});
}
}
exports.Runs = Runs;
Runs.OutputItems = output_items_1.OutputItems;
//# sourceMappingURL=runs.js.map
landvex/node_modules/openai/resources/evals/runs/runs.js.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"runs.js","sourceRoot":"","sources":["../../../src/resources/evals/runs/runs.ts"],"names":[],"mappings":";AAAA,sFAAsF;;;;AAEtF,wDAAqD;AAKrD,0EAAiD;AACjD,oDAOwB;AAExB,4DAA0F;AAE1F,0DAAoD;AAEpD;;GAEG;AACH,MAAa,IAAK,SAAQ,sBAAW;IAArC;;QACE,gBAAW,GAA+B,IAAI,cAAc,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAkEzF,CAAC;IAhEC;;;;OAIG;IACH,MAAM,CAAC,MAAc,EAAE,IAAqB,EAAE,OAAwB;QACpE,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAA,WAAI,EAAA,UAAU,MAAM,OAAO,EAAE;YACpD,IAAI;YACJ,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,QAAQ,CACN,KAAa,EACb,MAAyB,EACzB,OAAwB;QAExB,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAA,WAAI,EAAA,UAAU,OAAO,SAAS,KAAK,EAAE,EAAE;YAC7D,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAI,CACF,MAAc,EACd,QAA0C,EAAE,EAC5C,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAA,WAAI,EAAA,UAAU,MAAM,OAAO,EAAE,CAAA,uBAA2B,CAAA,EAAE;YACvF,KAAK;YACL,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAa,EAAE,MAAuB,EAAE,OAAwB;QACrE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAA,WAAI,EAAA,UAAU,OAAO,SAAS,KAAK,EAAE,EAAE;YAChE,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAa,EAAE,MAAuB,EAAE,OAAwB;QACrE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAA,WAAI,EAAA,UAAU,OAAO,SAAS,KAAK,EAAE,EAAE;YAC9D,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;CACF;AAnED,oBAmEC;AA8sFD,IAAI,CAAC,WAAW,GAAG,0BAAW,CAAC"}
landvex/node_modules/openai/resources/evals/runs/runs.mjs
Generated Vendored
+69
View File
@@ -0,0 +1,69 @@
// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
import { APIResource } from "../../../core/resource.mjs";
import * as OutputItemsAPI from "./output-items.mjs";
import { OutputItems, } from "./output-items.mjs";
import { CursorPage } from "../../../core/pagination.mjs";
import { path } from "../../../internal/utils/path.mjs";
/**
* Manage and run evals in the OpenAI platform.
*/
export class Runs extends APIResource {
constructor() {
super(...arguments);
this.outputItems = new OutputItemsAPI.OutputItems(this._client);
}
/**
* Kicks off a new run for a given evaluation, specifying the data source, and what
* model configuration to use to test. The datasource will be validated against the
* schema specified in the config of the evaluation.
*/
create(evalID, body, options) {
return this._client.post(path `/evals/${evalID}/runs`, {
body,
...options,
__security: { bearerAuth: true },
});
}
/**
* Get an evaluation run by ID.
*/
retrieve(runID, params, options) {
const { eval_id } = params;
return this._client.get(path `/evals/${eval_id}/runs/${runID}`, {
...options,
__security: { bearerAuth: true },
});
}
/**
* Get a list of runs for an evaluation.
*/
list(evalID, query = {}, options) {
return this._client.getAPIList(path `/evals/${evalID}/runs`, (CursorPage), {
query,
...options,
__security: { bearerAuth: true },
});
}
/**
* Delete an eval run.
*/
delete(runID, params, options) {
const { eval_id } = params;
return this._client.delete(path `/evals/${eval_id}/runs/${runID}`, {
...options,
__security: { bearerAuth: true },
});
}
/**
* Cancel an ongoing evaluation run.
*/
cancel(runID, params, options) {
const { eval_id } = params;
return this._client.post(path `/evals/${eval_id}/runs/${runID}`, {
...options,
__security: { bearerAuth: true },
});
}
}
Runs.OutputItems = OutputItems;
//# sourceMappingURL=runs.mjs.map
landvex/node_modules/openai/resources/evals/runs/runs.mjs.map
Generated Vendored
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"runs.mjs","sourceRoot":"","sources":["../../../src/resources/evals/runs/runs.ts"],"names":[],"mappings":"AAAA,sFAAsF;AAEtF,OAAO,EAAE,WAAW,EAAE,mCAA+B;AAKrD,OAAO,KAAK,cAAc,2BAAuB;AACjD,OAAO,EAML,WAAW,GACZ,2BAAuB;AAExB,OAAO,EAAE,UAAU,EAAsC,qCAAiC;AAE1F,OAAO,EAAE,IAAI,EAAE,yCAAqC;AAEpD;;GAEG;AACH,MAAM,OAAO,IAAK,SAAQ,WAAW;IAArC;;QACE,gBAAW,GAA+B,IAAI,cAAc,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAkEzF,CAAC;IAhEC;;;;OAIG;IACH,MAAM,CAAC,MAAc,EAAE,IAAqB,EAAE,OAAwB;QACpE,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAA,UAAU,MAAM,OAAO,EAAE;YACpD,IAAI;YACJ,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,QAAQ,CACN,KAAa,EACb,MAAyB,EACzB,OAAwB;QAExB,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA,UAAU,OAAO,SAAS,KAAK,EAAE,EAAE;YAC7D,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAI,CACF,MAAc,EACd,QAA0C,EAAE,EAC5C,OAAwB;QAExB,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAA,UAAU,MAAM,OAAO,EAAE,CAAA,UAA2B,CAAA,EAAE;YACvF,KAAK;YACL,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAa,EAAE,MAAuB,EAAE,OAAwB;QACrE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAA,UAAU,OAAO,SAAS,KAAK,EAAE,EAAE;YAChE,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAa,EAAE,MAAuB,EAAE,OAAwB;QACrE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;QAC3B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAA,UAAU,OAAO,SAAS,KAAK,EAAE,EAAE;YAC9D,GAAG,OAAO;YACV,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACjC,CAAC,CAAC;IACL,CAAC;CACF;AA8sFD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC"}