Evaluations
Evaluate models on a dataset
Evaluate models on the evaluation rows of a dataset’s latest version. Each queued
evaluation records the model it scores (model_id, model_label), for a family the
model it served then, and the family or model by its number it was asked by (named).
It scores that model to its end, even if the family moves or is switched off meanwhile.
A model evaluated on that version already, or being evaluated, is reused as it stands; a
failed or cancelled one runs again only through
POST /v1/evaluations/{evaluation_id}/retry.
POST
/
v1
/
evaluations
Evaluate models on a dataset
curl --request POST \
--url https://api.sqwish.ai/v1/evaluations \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"dataset_id": "<string>",
"models": [
"<string>"
]
}
'import requests
url = "https://api.sqwish.ai/v1/evaluations"
payload = {
"dataset_id": "<string>",
"models": ["<string>"]
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({dataset_id: '<string>', models: ['<string>']})
};
fetch('https://api.sqwish.ai/v1/evaluations', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));{
"dataset_id": "<string>",
"data": [
{
"id": "<string>",
"dataset_id": "<string>",
"dataset_version": 123,
"model_id": "<string>",
"model_sha256": "<string>",
"model_label": "<string>",
"model_name": "<string>",
"version": 123,
"status": "queued",
"created_at": 123,
"updated_at": 123,
"generation": 123,
"training_job_id": "<string>",
"dataset_sha256": "<string>",
"evaluation_sha256": "<string>",
"method": "sft",
"evaluation_rows": 123,
"decisions": 123,
"completed_rows": 123,
"metrics": {},
"error": {},
"named": "<string>"
}
],
"queued": 123,
"reused": 123
}{
"error": {
"code": "<string>",
"message": "<string>",
"retryable": true,
"request_id": "<string>",
"details": [
{
"path": [
"<string>"
],
"message": "<string>",
"type": "<string>"
}
]
}
}{
"error": {
"code": "<string>",
"message": "<string>",
"retryable": true,
"request_id": "<string>",
"details": [
{
"path": [
"<string>"
],
"message": "<string>",
"type": "<string>"
}
]
}
}Authorizations
Create an API key in the console.
Body
application/json
Pattern:
^ds_[a-f0-9]{24}$What to evaluate, each named as a call names it: a family, such as router, evaluated with the model it serves when the evaluation is queued; a model by its number, such as router@4; or a base model's id. Each must answer calls, so one switched off is refused.
Required array length:
1 - 16 elementsEvaluate models on a dataset
curl --request POST \
--url https://api.sqwish.ai/v1/evaluations \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"dataset_id": "<string>",
"models": [
"<string>"
]
}
'import requests
url = "https://api.sqwish.ai/v1/evaluations"
payload = {
"dataset_id": "<string>",
"models": ["<string>"]
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({dataset_id: '<string>', models: ['<string>']})
};
fetch('https://api.sqwish.ai/v1/evaluations', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));{
"dataset_id": "<string>",
"data": [
{
"id": "<string>",
"dataset_id": "<string>",
"dataset_version": 123,
"model_id": "<string>",
"model_sha256": "<string>",
"model_label": "<string>",
"model_name": "<string>",
"version": 123,
"status": "queued",
"created_at": 123,
"updated_at": 123,
"generation": 123,
"training_job_id": "<string>",
"dataset_sha256": "<string>",
"evaluation_sha256": "<string>",
"method": "sft",
"evaluation_rows": 123,
"decisions": 123,
"completed_rows": 123,
"metrics": {},
"error": {},
"named": "<string>"
}
],
"queued": 123,
"reused": 123
}{
"error": {
"code": "<string>",
"message": "<string>",
"retryable": true,
"request_id": "<string>",
"details": [
{
"path": [
"<string>"
],
"message": "<string>",
"type": "<string>"
}
]
}
}{
"error": {
"code": "<string>",
"message": "<string>",
"retryable": true,
"request_id": "<string>",
"details": [
{
"path": [
"<string>"
],
"message": "<string>",
"type": "<string>"
}
]
}
}