asqi_manifest.schema

asqi_manifest.schema.json

{
  "$defs": {
    "InputParameter": {
      "description": "Defines a parameter that can be passed to the test container.",
      "properties": {
        "name": {
          "title": "Name",
          "type": "string"
        },
        "type": {
          "enum": [
            "string",
            "integer",
            "float",
            "boolean",
            "list",
            "object"
          ],
          "title": "Type",
          "type": "string"
        },
        "required": {
          "default": false,
          "title": "Required",
          "type": "boolean"
        },
        "description": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Description"
        }
      },
      "required": [
        "name",
        "type"
      ],
      "title": "InputParameter",
      "type": "object"
    },
    "OutputArtifact": {
      "description": "Defines a file artifact generated by the test container.",
      "properties": {
        "name": {
          "title": "Name",
          "type": "string"
        },
        "path": {
          "title": "Path",
          "type": "string"
        },
        "description": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Description"
        }
      },
      "required": [
        "name",
        "path"
      ],
      "title": "OutputArtifact",
      "type": "object"
    },
    "OutputMetric": {
      "description": "Defines a metric that will be present in the test container's output.",
      "properties": {
        "name": {
          "title": "Name",
          "type": "string"
        },
        "type": {
          "enum": [
            "string",
            "integer",
            "float",
            "boolean",
            "list",
            "object"
          ],
          "title": "Type",
          "type": "string"
        },
        "description": {
          "anyOf": [
            {
              "type": "string"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "title": "Description"
        }
      },
      "required": [
        "name",
        "type"
      ],
      "title": "OutputMetric",
      "type": "object"
    },
    "SUTSupport": {
      "description": "Defines a type of SUT the container can test.",
      "properties": {
        "type": {
          "description": "The SUT type, e.g., 'llm_api' or 'rest_api'.",
          "title": "Type",
          "type": "string"
        },
        "required_config": {
          "anyOf": [
            {
              "items": {
                "type": "string"
              },
              "type": "array"
            },
            {
              "type": "null"
            }
          ],
          "default": null,
          "description": "List of config keys required by the entrypoint for this SUT type.",
          "title": "Required Config"
        }
      },
      "required": [
        "type"
      ],
      "title": "SUTSupport",
      "type": "object"
    }
  },
  "description": "Schema for the manifest.yaml file inside a test container.",
  "properties": {
    "name": {
      "description": "The canonical name for the test framework.",
      "title": "Name",
      "type": "string"
    },
    "version": {
      "title": "Version",
      "type": "string"
    },
    "description": {
      "anyOf": [
        {
          "type": "string"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Description"
    },
    "supported_suts": {
      "description": "Declares which SUT types this framework can handle.",
      "items": {
        "$ref": "#/$defs/SUTSupport"
      },
      "title": "Supported Suts",
      "type": "array"
    },
    "input_schema": {
      "default": [],
      "description": "Defines the schema for the user-provided 'params' object.",
      "items": {
        "$ref": "#/$defs/InputParameter"
      },
      "title": "Input Schema",
      "type": "array"
    },
    "output_metrics": {
      "anyOf": [
        {
          "items": {
            "type": "string"
          },
          "type": "array"
        },
        {
          "items": {
            "$ref": "#/$defs/OutputMetric"
          },
          "type": "array"
        }
      ],
      "default": [],
      "description": "Defines expected high-level metrics in the final JSON output. Can be a simple list of strings or detailed metric definitions.",
      "title": "Output Metrics"
    },
    "output_artifacts": {
      "anyOf": [
        {
          "items": {
            "$ref": "#/$defs/OutputArtifact"
          },
          "type": "array"
        },
        {
          "type": "null"
        }
      ],
      "default": null,
      "title": "Output Artifacts"
    }
  },
  "required": [
    "name",
    "version",
    "supported_suts"
  ],
  "title": "ASQI Manifest",
  "type": "object",
  "$schema": "https://json-schema.org/draft/2020-12/schema"
}