> ## Documentation Index
> Fetch the complete documentation index at: https://docs.erdo.ai/llms.txt
> Use this file to discover all available pages before exploring further.

# CreateEvalSuiteAPI mirrors erdo_create_eval_suite.




## OpenAPI

````yaml /api/openapi.json post /v1/evals/suites
openapi: 3.0.0
info:
  title: Erdo API
  description: >-
    Erdo's REST API: query and write datasets, run agents, manage threads,
    integrations, pages, evals, workstreams, and experiments. Authenticate with
    a Bearer API key (erdo_api_...) or scoped token (erdo_token_...).
  version: '2026-07-25'
servers:
  - url: https://api.erdo.ai
    description: Production
security:
  - bearerAuth: []
paths:
  /v1/evals/suites:
    post:
      summary: |
        CreateEvalSuiteAPI mirrors erdo_create_eval_suite.
      operationId: POST:mcp.CreateEvalSuiteAPI
      requestBody:
        content:
          application/json:
            schema:
              properties:
                agent_key:
                  type: string
                cases:
                  items:
                    $ref: '#/components/schemas/mcp.EvalCaseInput'
                  type: array
                cron_enabled:
                  type: boolean
                description:
                  type: string
                evaluate_artifact:
                  type: boolean
                judge_model:
                  type: string
                name:
                  type: string
                pass_threshold:
                  type: number
                slug:
                  type: string
              type: object
      responses:
        '200':
          content:
            application/json:
              schema:
                properties:
                  cases:
                    items:
                      $ref: '#/components/schemas/types.EvalCase'
                    type: array
                  suite:
                    $ref: '#/components/schemas/types.EvalSuite'
                type: object
          description: Success response
        default:
          $ref: '#/components/responses/APIError'
components:
  schemas:
    mcp.EvalCaseInput:
      description: >
        brief — encode the variation in it (voice agent id, lead forms,
        carousel, dataset slugs). The rubric criteria are the scoring lenses.
      properties:
        evaluators:
          items:
            $ref: '#/components/schemas/types.Evaluator'
          type: array
        input:
          type: string
        name:
          type: string
        rubric:
          items:
            $ref: '#/components/schemas/types.RubricCriterion'
          type: array
        setup_messages:
          description: >
            thread, as the same real agent — for multi-step user flows (e.g.
            \["Create a voice concierge widget for Lumen Yoga"] then input
            "Build the landing page wired to that concierge"). The judge scores
            the result of \`input\`.
          items:
            type: string
          title: >
            SetupMessages are optional turns run BEFORE the evaluated input, in
            the same
          type: array
        tags:
          items:
            type: string
          type: array
      title: >
        EvalCaseInput is one case in a suite. For artifact suites the input is
        the page
      type: object
    types.EvalCase:
      properties:
        context:
          type: object
        created_at:
          format: date-time
          type: string
        evaluators:
          items:
            $ref: '#/components/schemas/types.Evaluator'
          type: array
        id:
          format: uuid
          type: string
        input:
          type: string
        name:
          type: string
        rubric:
          items:
            $ref: '#/components/schemas/types.RubricCriterion'
          type: array
        suite_id:
          format: uuid
          type: string
        tags:
          items:
            type: string
          type: array
        updated_at:
          format: date-time
          type: string
      title: |
        Case is one prompt + rubric the judge scores against.
      type: object
    types.EvalSuite:
      properties:
        agent_key:
          type: string
        created_at:
          format: date-time
          type: string
        created_by_user_id:
          format: uuid
          type: string
        cron_enabled:
          description: >
            suites build a full page + render-judge per case, so they default to
            on-demand.
          title: >
            CronEnabled: the daily all-orgs cron skips suites with this false.
            Artifact
          type: boolean
        description:
          type: string
        evaluate_artifact:
          description: >
            built during the run and judges the rendered page (screenshots +
            interaction) rather than the agent's text output. The rubric
            criteria carry the lenses.
          title: >
            EvaluateArtifact: when true, the runner resolves the HTML artifact
            the agent
          type: boolean
        id:
          format: uuid
          type: string
        judge_model:
          type: string
        name:
          type: string
        organization_id:
          format: uuid
          type: string
        pass_threshold:
          type: number
        slug:
          description: |
            the suite (never the UUID). Derived from name at create time.
          title: >
            Slug is the stable, human-readable identifier MCP/CLI tools use to
            reference
          type: string
        updated_at:
          format: date-time
          type: string
      title: |
        EvalSuite groups cases that target a single agent.
      type: object
    types.Evaluator:
      description: >
        llm\_rubric evaluator (from the case rubric); set evaluators explicitly
        to mix cheap deterministic checks with LLM judging.

          - "llm\_rubric": an LLM scores the agent output (or the rendered page when the suite has evaluate\_artifact) against Rubric.
          - "script": a JS function \`evaluate(ctx)\` run via jsexec returns {score:0-5, passed:bool, reasoning:string}. ctx = {input, output, artifact, data\_store}. No LLM tokens — use it to assert on output/artifact/persisted data deterministically.
      properties:
        rubric:
          items:
            $ref: '#/components/schemas/types.RubricCriterion'
          type: array
        script:
          type: string
        type:
          type: string
        weight:
          type: number
      title: |
        Evaluator is one scoring step for a case. Cases default to a single
      type: object
    types.RubricCriterion:
      description: |
        0-5 per criterion; the final score is a weight-normalised average.
      properties:
        criterion:
          type: string
        weight:
          type: number
      title: >
        RubricCriterion is one scoring dimension for an eval case. The judge
        scores
      type: object
  responses:
    APIError:
      content:
        application/json:
          schema:
            externalDocs:
              url: https://pkg.go.dev/encore.dev/beta/errs#Error
            properties:
              code:
                description: Error code
                example: not_found
                externalDocs:
                  url: https://pkg.go.dev/encore.dev/beta/errs#ErrCode
                type: string
              details:
                description: Error details
                type: object
              message:
                description: Error message
                type: string
            title: APIError
            type: object
      description: Error response
  securitySchemes:
    bearerAuth:
      type: http
      scheme: bearer
      description: An Erdo API key (erdo_api_...) or scoped token (erdo_token_...).

````