Nerf Watch

Latest public Nerf Watch scores, previous runs, changes and baselines for at most 100 models, ordered by provider, name and model id. No arguments. Histories are available through get_nerfwatch_history. Scores and changes are rounded to two decimals; comparisons are within one suite version. A lower score is a lower score on this suite, not proof that a provider changed a model. Unofficial, not affiliated with OpenAI or Anthropic. Source: https://tokenusage.dev/nerfwatch

{
  "type": "object",
  "properties": {},
  "additionalProperties": false
}

{
  "type": "object",
  "properties": {
    "source": {
      "type": "string",
      "enum": [
        "https://tokenusage.dev/nerfwatch"
      ],
      "description": "The fixed credit URL, https://tokenusage.dev/nerfwatch. When showing the data to other people, show Data: tokenusage.dev linked to this URL on the same screen or in the same message. Personal use needs no visible credit. API and MCP relays must retain source and pass on this credit condition."
    },
    "generatedAt": {
      "type": "string",
      "description": "When the cached data was read, ISO 8601 UTC; the cache TTL is 300 seconds from read start."
    },
    "dataUpdatedAt": {
      "type": [
        "string",
        "null"
      ],
      "description": "When the newest retained run was stored, ISO 8601 UTC; null without runs. This is distinct from response generation time."
    },
    "baselineDays": {
      "type": "integer",
      "minimum": 1,
      "description": "The UTC-day window before each latest run used for its baseline; this is the returned setting, not a fixed client assumption."
    },
    "minBaselineRuns": {
      "type": "integer",
      "minimum": 1,
      "description": "The minimum number of same-suite runs needed for a baseline; this is the returned setting."
    },
    "flatPoints": {
      "type": "number",
      "minimum": 0,
      "description": "An absolute change versus baseline at or below this returned setting is flat."
    },
    "models": {
      "type": "array",
      "maxItems": 100,
      "description": "Models in source order: provider, name and model id; empty before the first run.",
      "items": {
        "type": "object",
        "additionalProperties": true,
        "properties": {
          "model": {
            "type": "string",
            "description": "The model id used by get_nerfwatch_history."
          },
          "name": {
            "type": "string",
            "description": "The model's published display name."
          },
          "provider": {
            "type": "string",
            "description": "The model's published provider name."
          },
          "latest": {
            "type": [
              "object",
              "null"
            ],
            "additionalProperties": true,
            "description": "The newest retained run, or null.",
            "properties": {
              "day": {
                "type": "string",
                "pattern": "^\\d{4}-\\d{2}-\\d{2}$",
                "description": "The run's UTC day, YYYY-MM-DD."
              },
              "score": {
                "type": "number",
                "minimum": 0,
                "maximum": 100,
                "description": "The suite percentage, rounded to two decimals."
              },
              "suiteVersion": {
                "type": "string",
                "description": "The suite version; score comparisons are within one version."
              }
            },
            "required": [
              "day",
              "score",
              "suiteVersion"
            ]
          },
          "previous": {
            "type": [
              "object",
              "null"
            ],
            "additionalProperties": true,
            "description": "The retained run before the latest, whatever its day or suite version, or null.",
            "properties": {
              "day": {
                "type": "string",
                "pattern": "^\\d{4}-\\d{2}-\\d{2}$",
                "description": "The run's UTC day, YYYY-MM-DD."
              },
              "score": {
                "type": "number",
                "minimum": 0,
                "maximum": 100,
                "description": "The suite percentage, rounded to two decimals."
              },
              "suiteVersion": {
                "type": "string",
                "description": "The suite version; score comparisons are within one version."
              }
            },
            "required": [
              "day",
              "score",
              "suiteVersion"
            ]
          },
          "change": {
            "type": [
              "number",
              "null"
            ],
            "minimum": -100,
            "maximum": 100,
            "description": "Latest minus previous in points, rounded to two decimals; null without both runs on the same suite version."
          },
          "baseline": {
            "type": [
              "number",
              "null"
            ],
            "minimum": 0,
            "maximum": 100,
            "description": "The mean same-suite score in the baseline window, rounded to two decimals; null with fewer than minBaselineRuns."
          },
          "baselineRuns": {
            "type": "integer",
            "minimum": 0,
            "description": "The number of same-suite runs in the baseline window, including when too few for a baseline."
          },
          "changeVsBaseline": {
            "type": [
              "number",
              "null"
            ],
            "minimum": -100,
            "maximum": 100,
            "description": "Latest minus baseline in points, rounded to two decimals; null without a baseline."
          },
          "trend": {
            "type": [
              "string",
              "null"
            ],
            "enum": [
              "down",
              "flat",
              "up",
              null
            ],
            "description": "Direction versus baseline: down below -flatPoints, up above flatPoints, flat otherwise; null without a baseline."
          }
        },
        "required": [
          "model",
          "name",
          "provider",
          "latest",
          "previous",
          "change",
          "baseline",
          "baselineRuns",
          "changeVsBaseline",
          "trend"
        ]
      }
    }
  },
  "required": [
    "source",
    "generatedAt",
    "dataUpdatedAt",
    "baselineDays",
    "minBaselineRuns",
    "flatPoints",
    "models"
  ],
  "additionalProperties": true
}

Retained public Nerf Watch score history for one model, oldest first, including each point's suite version. The required model id is available from get_nerfwatch_scores. Missing days are missing, not zero; comparisons are within one suite version. A lower score is a lower score on this suite, not proof that a provider changed a model. Unofficial, not affiliated with OpenAI or Anthropic. Source: https://tokenusage.dev/nerfwatch

{
  "type": "object",
  "properties": {
    "model": {
      "type": "string",
      "pattern": "^[a-z0-9][a-z0-9._-]{0,63}$",
      "description": "A model id from get_nerfwatch_scores: 1 to 64 lowercase letters, digits, dots, underscores or hyphens, starting with a letter or digit."
    }
  },
  "required": [
    "model"
  ],
  "additionalProperties": false
}

{
  "type": "object",
  "properties": {
    "source": {
      "type": "string",
      "enum": [
        "https://tokenusage.dev/nerfwatch"
      ],
      "description": "The fixed credit URL, https://tokenusage.dev/nerfwatch. When showing the data to other people, show Data: tokenusage.dev linked to this URL on the same screen or in the same message. Personal use needs no visible credit. API and MCP relays must retain source and pass on this credit condition."
    },
    "generatedAt": {
      "type": "string",
      "description": "When the cached data was read, ISO 8601 UTC; the cache TTL is 300 seconds from read start."
    },
    "dataUpdatedAt": {
      "type": [
        "string",
        "null"
      ],
      "description": "When the newest retained run across the dataset was stored, ISO 8601 UTC; null without runs. This is dataset metadata, not a timestamp for this model alone."
    },
    "historyDays": {
      "type": "integer",
      "minimum": 1,
      "description": "The returned retention bound per model; clients read this setting rather than assuming 366."
    },
    "model": {
      "type": "string",
      "description": "The selected model id."
    },
    "name": {
      "type": "string",
      "description": "The selected model's published display name."
    },
    "provider": {
      "type": "string",
      "description": "The selected model's published provider name."
    },
    "history": {
      "type": "array",
      "description": "Retained points, oldest first, including older suite versions; days without a run are missing, not zero.",
      "items": {
        "type": "object",
        "additionalProperties": true,
        "properties": {
          "day": {
            "type": "string",
            "pattern": "^\\d{4}-\\d{2}-\\d{2}$",
            "description": "The run's UTC day, YYYY-MM-DD."
          },
          "score": {
            "type": "number",
            "minimum": 0,
            "maximum": 100,
            "description": "The suite percentage, rounded to two decimals."
          },
          "suiteVersion": {
            "type": "string",
            "description": "The suite version; score comparisons are within one version."
          }
        },
        "required": [
          "day",
          "score",
          "suiteVersion"
        ]
      }
    }
  },
  "required": [
    "source",
    "generatedAt",
    "dataUpdatedAt",
    "historyDays",
    "model",
    "name",
    "provider",
    "history"
  ],
  "additionalProperties": true
}