{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://assess.go.phpboyscout.uk/reference/schema/assess.schema.json",
  "title": "assess.yaml",
  "description": "go/assess's configuration, spec 0001 D3.",
  "type": "object",
  "additionalProperties": false,
  "required": ["version", "dataset", "split", "scorer", "models", "sampling", "thinking", "serving", "concurrency"],
  "properties": {
    "version": {"const": 1},
    "server": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "url": {"type": "string", "minLength": 1},
        "manifest": {"type": "string", "minLength": 1},
        "toolCallParser": {"type": "string", "minLength": 1},
        "reasoningParser": {"type": "string", "minLength": 1},
        "insecure": {"type": "boolean"}
      }
    },
    "dataset": {
      "type": "object",
      "additionalProperties": false,
      "required": ["kind", "path", "sha256", "source"],
      "properties": {
        "kind": {"const": "xlam-function-calling"},
        "path": {"type": "string", "minLength": 1},
        "sha256": {"type": "string", "pattern": "^[0-9a-fA-F]{64}$"},
        "source": {
          "type": "object",
          "additionalProperties": false,
          "required": ["repo", "revision"],
          "properties": {
            "repo": {"type": "string", "minLength": 1},
            "revision": {"type": "string", "minLength": 1}
          }
        }
      }
    },
    "split": {
      "type": "object",
      "additionalProperties": false,
      "required": ["seed", "size"],
      "properties": {
        "seed": {"type": "string"},
        "size": {"type": "integer", "minimum": 1, "maximum": 2147483647}
      }
    },
    "scorer": {
      "type": "object",
      "additionalProperties": false,
      "required": ["name"],
      "properties": {
        "name": {"type": "string", "minLength": 1},
        "ordering": {"enum": ["any", "strict"]}
      }
    },
    "prompt": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "system": {"type": "string"}
      }
    },
    "models": {
      "type": "object",
      "additionalProperties": false,
      "required": ["baseline"],
      "properties": {
        "baseline": {"type": "string", "minLength": 1},
        "adapters": {
          "type": "array",
          "items": {
            "type": "object",
            "additionalProperties": false,
            "required": ["name"],
            "properties": {
              "name": {"type": "string", "minLength": 1},
              "path": {"type": "string", "minLength": 1}
            }
          }
        }
      }
    },
    "sampling": {
      "type": "object",
      "additionalProperties": false,
      "required": ["temperature", "topP", "seed", "maxTokens"],
      "properties": {
        "temperature": {"type": "number", "minimum": 0},
        "topP": {"type": "number", "exclusiveMinimum": 0, "maximum": 1},
        "seed": {"type": "integer", "minimum": -9223372036854775808, "maximum": 9223372036854775807},
        "maxTokens": {"type": "integer", "minimum": 1, "maximum": 2147483647}
      }
    },
    "thinking": {
      "type": "object",
      "additionalProperties": false,
      "required": ["mode", "templateKey"],
      "properties": {
        "mode": {"enum": ["off", "on"]},
        "templateKey": {"type": "string", "minLength": 1}
      }
    },
    "serving": {
      "type": "object",
      "additionalProperties": false,
      "required": ["probe"],
      "properties": {
        "batchInvariant": {"enum": ["required", "asserted", "not-required"]},
        "probe": {"type": "integer", "minimum": 0, "maximum": 2147483647}
      }
    },
    "concurrency": {"type": "integer", "minimum": 1, "maximum": 2147483647}
  }
}