AI Apps That Work free tools
Real cost per AI result, LLM prices, switch costs, shutdown dates, A/B test sizes and judge checks.
Sollte ich dies verwenden
Qualität und Sicherheit
Befunde (3)
- HIGH
- MEDIUMin build_judge_prompt
- INFOin build_judge_prompt
Basierend auf einer automatisierten Analyse der Tool-Definitionen und der Einhaltung des Protokolls.
Kontextkosten
Dies ist die ungefähre Anzahl der Tokens, die jedes Mal verbraucht werden, wenn die Tools des Servers in den Kontext eines Modells geladen werden. Höhere Werte verringern die Aufmerksamkeit, die für andere Aufgaben verfügbar ist.
Installieren
Installation mit einem Klick
Fügen Sie dies Ihrer Datei `claude_desktop_config.json` hinzu:
{
"mcpServers": {
"tools": {
"url": "https://aiappsthatwork.com/mcp"
}
}
}Remote-Endpunkte
https://aiappsthatwork.com/mcpstreamable-httpWas es kann
Tool-Inventar
Tools (8)
🟢real_cost_per_result(steps, workedFirstTimePct, requestsPerMonth, pricing)
Prices the AI steps behind one request from current list prices (LLM tokens, images, video seconds or a fixed amount) and returns the cost per request, the real cost per result that worked first time (cost per request divided by the worked-first-time share), the monthly spend wasted on results that didn't work, and credit-pricing numbers: margin per result and the credits per request needed to break even or reach a target margin.
Eingabe-Schema
{
"type": "object",
"properties": {
"steps": {
"minItems": 1,
"maxItems": 4,
"type": "array",
"items": {
"oneOf": [
{
"type": "object",
"properties": {
"kind": {
"type": "string",
"const": "llm"
},
"model": {
"type": "string",
"minLength": 1,
"description": "LLM model id from /models.json, for example anthropic/claude-haiku-5.5"
},
"inputTokens": {
"type": "integer",
"minimum": 0,
"maximum": 10000000,
"description": "Input tokens per request"
},
"outputTokens": {
"type": "integer",
"minimum": 0,
"maximum": 10000000,
"description": "Output tokens per request"
},
"cachedInputPct": {
"default": 0,
"description": "Share of input tokens billed at the provider's cached-input price, 0 to 100",
"type": "number",
"minimum": 0,
"maximum": 100
},
"note": {
"description": "What the step does, for your own reference",
"type": "string",
"maxLength": 120
}
},
"required": [
"kind",
"model",
"inputTokens",
"outputTokens"
]
},
{
"type": "object",
"properties": {
"kind": {
"type": "string",
"const": "image"
},
"model": {
"type": "string",
"minLength": 1,
"description": "Image model id from /models.json, for example google/nano-banana-2.1"
},
"variant": {
"type": "string",
"minLength": 1,
"description": "Quality and size label exactly as listed for the model in /models.json"
},
"images": {
"default": 1,
"description": "Images per request",
"type": "integer",
"minimum": 1,
"maximum": 100
},
"note": {
"description": "What the step does, for your own reference",
"type": "string",
"maxLength": 120
}
},
"required": [
"kind",
"model",
"variant"
]
},
{
"type": "object",
"properties": {
"kind": {
"type": "string",
"const": "video"
},
"model": {
"type": "string",
"minLength": 1,
"description": "Video model id from /models.json, for example google/veo-3.1-fast"
},
"variant": {
"type": "string",
"minLength": 1,
"description": "Resolution and audio label exactly as listed for the model in /models.json"
},
"seconds": {
"type": "number",
"exclusiveMinimum": 0,
"maximum": 600,
"description": "Seconds of video per request"
},
"note": {
"description": "What the step does, for your own reference",
"type": "string",
"maxLength": 120
}
},
"required": [
"kind",
"model",
"variant",
"seconds"
]
},
{
"type": "object",
"properties": {
"kind": {
"type": "string",
"const": "custom"
},
"usd": {
"type": "number",
"minimum": 0,
"maximum": 1000,
"description": "What this step costs per request, in USD"
},
"label": {
"description": "What the step is, for example a speech-to-text API",
"type": "string",
"maxLength": 120
}
},
"required": [
"kind",
"usd"
]
}
]
},
"description": "The AI steps behind one request, priced from /models.json. Their costs add up to the cost per request."
},
"workedFirstTimePct": {
"type": "number",
"minimum": 1,
"maximum": 100,
"description": "Share of results that worked first time: no try again, undo, fix, re-ask or overrule. 1 to 100."
},
"requestsPerMonth": {
"type": "number",
"minimum": 0,
"maximum": 1000000000,
"description": "Requests per month, try-agains included"
},
"pricing": {
"description": "Credit pricing. Defaults: $10 for 1,000 credits, 1 credit per request, a 2.9% + $0.30 fee and a 70% target margin.",
"default": {},
"type": "object",
"properties": {
"packPriceUsd": {
"default": 10,
"description": "What a user pays for one credit pack, USD",
"type": "number",
"exclusiveMinimum": 0
},
"packCredits": {
"default": 1000,
"description": "Credits in that pack",
"type": "number",
"exclusiveMinimum": 0
},
"creditsPerRequest": {
"default": 1,
"description": "Credits charged each time the AI runs",
"type": "number",
"exclusiveMinimum": 0
},
"chargeOnlyWorked": {
"default": false,
"description": "True if users are charged only for results that worked",
"type": "boolean"
},
"feePct": {
"default": 2.9,
"description": "Payment processor fee, percent of the pack price",
"type": "number",
"minimum": 0,
"exclusiveMaximum": 100
},
"feeFixedUsd": {
"default": 0.3,
"description": "Payment processor fixed fee per pack, USD",
"type": "number",
"minimum": 0
},
"targetMarginPct": {
"default": 70,
"description": "Gross margin wanted on each result that worked, percent",
"type": "number",
"minimum": 0,
"exclusiveMaximum": 100
}
}
}
},
"required": [
"steps",
"workedFirstTimePct",
"requestsPerMonth"
],
"$schema": "https://json-schema.org/draft/2020-12/schema"
}Ausgabe-Schema
{
"type": "object",
"properties": {
"costPerRequestUsd": {
"type": "number",
"description": "Sum of the steps' costs for one request"
},
"realCostPerResultUsd": {
"type": "number",
"description": "Cost per request divided by the worked-first-time share: what each result that worked costs, counting failed tries"
},
"requestsPerResult": {
"type": "number",
"description": "Requests paid for per result that worked"
},
"monthlySpendUsd": {
"type": "number"
},
"wastedSpendPerMonthUsd": {
"type": "number",
"description": "Monthly spend on results that didn't work first time"
},
"steps": {
"type": "array",
"items": {
"type": "object",
"properties": {
"usd": {
"type": "number",
"description": "This step's cost per request"
},
"detail": {
"type": "string",
"description": "How the cost was worked out"
},
"model": {
"type": [
"string",
"null"
]
},
"priceSource": {
"description": "The provider's pricing page",
"type": [
"string",
"null"
]
},
"shutdown": {
"anyOf": [
{
"type": "object",
"properties": {
"date": {
"type": "string"
},
"replacement": {
"type": [
"string",
"null"
]
}
},
"required": [
"date",
"replacement"
],
"additionalProperties": false
},
{
"type": "null"
}
],
"description": "The model's announced API shutdown, if any"
}
},
"required": [
"usd",
"detail",
"model",
"priceSource",
"shutdown"
],
"additionalProperties": false
}
},
"credits": {
"type": "object",
"properties": {
"netPerCreditUsd": {
"type": "number",
"description": "What one credit brings in after payment fees"
},
"revenuePerResultUsd": {
"type": "number",
"description": "Net revenue per result that worked"
},
"marginPct": {
"description": "Gross margin per result that worked; null when revenue is zero",
"type": [
"number",
"null"
]
},
"userPricePerResultUsd": {
"type": "number",
"description": "What the user pays per result that worked, before fees"
},
"breakEvenCreditsPerRequest": {
"type": "number",
"description": "Fewest whole credits per request that don't lose money"
},
"suggestedCreditsPerRequest": {
"type": "number",
"description": "Fewest whole credits per request that reach the target margin"
},
"targetMarginPct": {
"type": "number"
}
},
"required": [
"netPerCreditUsd",
"revenuePerResultUsd",
"marginPct",
"userPricePerResultUsd",
"breakEvenCreditsPerRequest",
"suggestedCreditsPerRequest",
"targetMarginPct"
],
"additionalProperties": false
},
"pricesChecked": {
"type": "string",
"description": "Date the model prices were last checked"
},
"summary": {
"type": "string"
},
"permalink": {
"type": "string",
"description": "A page on this site that shows this result, for a person to open"
}
},
"required": [
"costPerRequestUsd",
"realCostPerResultUsd",
"requestsPerResult",
"monthlySpendUsd",
"wastedSpendPerMonthUsd",
"steps",
"credits",
"pricesChecked",
"summary",
"permalink"
],
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": false
}🟢model_switch_cost(from, to)
Compares two priced LLMs on six common task shapes (support answer, invoice extraction, lead scoring, email draft, translation, search re-ranking). Returns list prices, cost per 1,000 requests on each model, the cost ratio, the break-even (how often the new model's results must work first time, relative to the current one, to cost no more per result that works) and the current model's announced shutdown.
Eingabe-Schema
{
"type": "object",
"properties": {
"from": {
"type": "string",
"minLength": 1,
"maxLength": 100,
"description": "The current model: an LLM id from /models.json (openai/gpt-4o), an OpenRouter id or the model's name"
},
"to": {
"type": "string",
"minLength": 1,
"maxLength": 100,
"description": "The new model: an LLM id from /models.json (openai/gpt-6-luna), an OpenRouter id or the model's name"
}
},
"required": [
"from",
"to"
],
"$schema": "https://json-schema.org/draft/2020-12/schema"
}Ausgabe-Schema
{
"type": "object",
"properties": {
"from": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"provider": {
"type": "string"
},
"inputPerMtokUsd": {
"type": "number"
},
"outputPerMtokUsd": {
"type": "number"
},
"cachedInputPerMtokUsd": {
"type": [
"number",
"null"
]
},
"priceSource": {
"type": "string"
},
"priceChecked": {
"type": "string",
"description": "Date this model's price was last confirmed (YYYY-MM-DD)"
},
"priceChanged": {
"description": "Date this model's price last changed; null if unchanged since tracking began",
"type": [
"string",
"null"
]
}
},
"required": [
"id",
"name",
"provider",
"inputPerMtokUsd",
"outputPerMtokUsd",
"cachedInputPerMtokUsd",
"priceSource",
"priceChecked",
"priceChanged"
],
"additionalProperties": false
},
"to": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"provider": {
"type": "string"
},
"inputPerMtokUsd": {
"type": "number"
},
"outputPerMtokUsd": {
"type": "number"
},
"cachedInputPerMtokUsd": {
"type": [
"number",
"null"
]
},
"priceSource": {
"type": "string"
},
"priceChecked": {
"type": "string",
"description": "Date this model's price was last confirmed (YYYY-MM-DD)"
},
"priceChanged": {
"description": "Date this model's price last changed; null if unchanged since tracking began",
"type": [
"string",
"null"
]
}
},
"required": [
"id",
"name",
"provider",
"inputPerMtokUsd",
"outputPerMtokUsd",
"cachedInputPerMtokUsd",
"priceSource",
"priceChecked",
"priceChanged"
],
"additionalProperties": false
},
"tokenFactor": {
"type": "number",
"description": "Tokens the new model counts for text that is 1 token on the current one, where the provider publishes it; otherwise 1"
},
"costChange": {
"type": "string",
"description": "The change in cost per request across the tasks"
},
"tasks": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"inputTokens": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"outputTokens": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"assumption": {
"type": "string",
"description": "What goes in and comes out; illustrative sizes, not measurements"
},
"fromCostPer1000Usd": {
"type": "number"
},
"toCostPer1000Usd": {
"type": "number"
},
"costRatio": {
"type": "number",
"description": "New cost divided by current cost per request"
},
"breakEven": {
"type": "string",
"description": "How often the new model's results must work first time, relative to the current model's, to cost no more per result that works"
}
},
"required": [
"id",
"name",
"inputTokens",
"outputTokens",
"assumption",
"fromCostPer1000Usd",
"toCostPer1000Usd",
"costRatio",
"breakEven"
],
"additionalProperties": false
}
},
"fromShutdown": {
"anyOf": [
{
"type": "object",
"properties": {
"date": {
"type": "string"
},
"replacement": {
"type": [
"string",
"null"
]
},
"autoRoutes": {
"type": "boolean"
},
"source": {
"type": "string"
}
},
"required": [
"date",
"replacement",
"autoRoutes",
"source"
],
"additionalProperties": false
},
{
"type": "null"
}
],
"description": "The current model's announced API shutdown, if any"
},
"note": {
"type": [
"string",
"null"
]
},
"switchTestUrl": {
"description": "The browser test that runs your own prompt on both models with your own API key",
"type": [
"string",
"null"
]
},
"pricesChecked": {
"type": "string"
},
"summary": {
"type": "string"
},
"permalink": {
"type": "string",
"description": "A page on this site that shows this result, for a person to open"
}
},
"required": [
"from",
"to",
"tokenFactor",
"costChange",
"tasks",
"fromShutdown",
"note",
"switchTestUrl",
"pricesChecked",
"summary",
"permalink"
],
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": false
}🟢model_prices(query, kind, limit)
Looks up list prices for LLMs (USD per million input, output and cached input tokens, and context size), image models (USD per image by quality and size) and video models (USD per second by resolution), with each price's source page, the date prices were checked and any announced API shutdown.
Eingabe-Schema
{
"type": "object",
"properties": {
"query": {
"description": "A model id such as 'gpt-4o' (only exact id matches, if any), or words that must all appear in the id, name or provider, such as 'claude sonnet'. Omitted: every model.",
"type": "string",
"maxLength": 100
},
"kind": {
"default": "all",
"description": "llm for text models priced per token, image per image, video per second",
"type": "string",
"enum": [
"llm",
"image",
"video",
"all"
]
},
"limit": {
"default": 25,
"description": "Most models to return",
"type": "integer",
"minimum": 1,
"maximum": 100
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema"
}Ausgabe-Schema
{
"type": "object",
"properties": {
"pricesChecked": {
"type": "string",
"description": "Date of the last full price check. Each model also has its own priceChecked and priceChanged."
},
"totalMatches": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"models": {
"type": "array",
"items": {
"type": "object",
"properties": {
"kind": {
"type": "string",
"enum": [
"llm",
"image",
"video"
]
},
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"provider": {
"type": "string"
},
"inputPerMtokUsd": {
"description": "LLMs: USD per million input tokens",
"type": [
"number",
"null"
]
},
"outputPerMtokUsd": {
"description": "LLMs: USD per million output tokens",
"type": [
"number",
"null"
]
},
"cachedInputPerMtokUsd": {
"description": "LLMs: USD per million cached input tokens, where offered",
"type": [
"number",
"null"
]
},
"contextTokens": {
"type": [
"number",
"null"
]
},
"variants": {
"anyOf": [
{
"type": "array",
"items": {
"type": "object",
"properties": {
"label": {
"type": "string"
},
"usdPerImage": {
"type": [
"number",
"null"
]
},
"usdPerSecond": {
"type": [
"number",
"null"
]
}
},
"required": [
"label",
"usdPerImage",
"usdPerSecond"
],
"additionalProperties": false
}
},
{
"type": "null"
}
],
"description": "Image and video models: price per quality, size or resolution"
},
"notes": {
"type": "string"
},
"priceSource": {
"type": "string"
},
"priceChecked": {
"type": "string",
"description": "Date this model's price was last confirmed against its source (YYYY-MM-DD)"
},
"priceChanged": {
"description": "Date this model's price last changed; null if unchanged since tracking began",
"type": [
"string",
"null"
]
},
"shutdown": {
"anyOf": [
{
"type": "object",
"properties": {
"date": {
"type": "string"
},
"replacement": {
"type": [
"string",
"null"
]
}
},
"required": [
"date",
"replacement"
],
"additionalProperties": false
},
{
"type": "null"
}
],
"description": "Announced API shutdown, if any"
},
"page": {
"type": "string",
"description": "The page on this site that lists this model"
}
},
"required": [
"kind",
"id",
"name",
"provider",
"inputPerMtokUsd",
"outputPerMtokUsd",
"cachedInputPerMtokUsd",
"contextTokens",
"variants",
"notes",
"priceSource",
"priceChecked",
"priceChanged",
"shutdown",
"page"
],
"additionalProperties": false
}
},
"permalink": {
"type": "string",
"description": "A page on this site that shows this result, for a person to open"
}
},
"required": [
"pricesChecked",
"totalMatches",
"models",
"permalink"
],
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": false
}🟢model_shutdowns(query, includePast, today)
Lists announced API shutdown dates for AI models from OpenAI, Anthropic, Google and other providers, with days left, the suggested replacement, the source page, and links to a cost comparison and a side-by-side test of the replacement where this site has them.
Eingabe-Schema
{
"type": "object",
"properties": {
"query": {
"description": "A model id such as 'gpt-4o' (only exact id matches, if any), or words that must all appear in the model ids or provider, such as 'anthropic'. Omitted: every model.",
"type": "string",
"maxLength": 100
},
"includePast": {
"default": false,
"description": "Include shutdowns whose date has passed",
"type": "boolean"
},
"today": {
"description": "Date to count days from, YYYY-MM-DD. Omitted: today in UTC.",
"type": "string",
"pattern": "^\\d{4}-\\d{2}-\\d{2}$"
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema"
}Ausgabe-Schema
{
"type": "object",
"properties": {
"today": {
"type": "string"
},
"listChecked": {
"type": "string",
"description": "Date the list was last checked against the providers' deprecation pages"
},
"count": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"shutdowns": {
"type": "array",
"items": {
"type": "object",
"properties": {
"model": {
"type": "string",
"description": "The model ids as the provider's deprecation page writes them"
},
"provider": {
"type": "string"
},
"shutdownDate": {
"type": "string",
"description": "When API calls naming the model stop working, YYYY-MM-DD"
},
"daysLeft": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991,
"description": "Negative once the date has passed"
},
"replacement": {
"description": "The provider's suggested replacement",
"type": [
"string",
"null"
]
},
"source": {
"type": "string"
},
"confidence": {
"type": "string"
},
"switchPage": {
"description": "This site's cost comparison for the move to the replacement",
"type": [
"string",
"null"
]
},
"switchTestUrl": {
"description": "The browser test with both models picked, run with your own API key",
"type": [
"string",
"null"
]
}
},
"required": [
"model",
"provider",
"shutdownDate",
"daysLeft",
"replacement",
"source",
"confidence",
"switchPage",
"switchTestUrl"
],
"additionalProperties": false
}
},
"permalink": {
"type": "string",
"description": "A page on this site that shows this result, for a person to open"
}
},
"required": [
"today",
"listChecked",
"count",
"shutdowns",
"permalink"
],
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": false
}🟢ab_test_sample_size(baselinePct, targetPct, confidencePct, powerPct, resultsPerDay)
Returns how many results each version of a prompt or model needs before a difference in how often results work first time is statistically real (two-proportion z-test, two-sided), the total across both versions, and how many days that takes at a given daily volume.
Eingabe-Schema
{
"type": "object",
"properties": {
"baselinePct": {
"type": "number",
"exclusiveMinimum": 0,
"exclusiveMaximum": 100,
"description": "Share of version A's (the current prompt or model) results that work first time, percent (above 0, below 100)"
},
"targetPct": {
"type": "number",
"exclusiveMinimum": 0,
"exclusiveMaximum": 100,
"description": "Share of version B's results that work first time, percent (above 0, below 100)"
},
"confidencePct": {
"default": 95,
"description": "Confidence level, percent. Two-sided.",
"type": "number",
"enum": [
90,
95,
99
]
},
"powerPct": {
"default": 80,
"description": "Chance of detecting a real difference of this size, percent",
"type": "number",
"enum": [
80,
90
]
},
"resultsPerDay": {
"description": "Results per day across both versions, split evenly. Gives the number of days.",
"type": "number",
"exclusiveMinimum": 0
}
},
"required": [
"baselinePct",
"targetPct"
],
"$schema": "https://json-schema.org/draft/2020-12/schema"
}Ausgabe-Schema
{
"type": "object",
"properties": {
"perVersion": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"total": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"days": {
"anyOf": [
{
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
{
"type": "null"
}
],
"description": "Null when resultsPerDay isn't given"
},
"baselinePct": {
"type": "number"
},
"targetPct": {
"type": "number"
},
"alpha": {
"type": "number",
"description": "False-positive rate, 1 minus the confidence level"
},
"power": {
"type": "number"
},
"method": {
"type": "string"
},
"otherLifts": {
"type": "array",
"items": {
"type": "object",
"properties": {
"liftPts": {
"type": "number"
},
"targetPct": {
"type": "number"
},
"perVersion": {
"anyOf": [
{
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
{
"type": "null"
}
]
},
"total": {
"anyOf": [
{
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
{
"type": "null"
}
]
},
"days": {
"anyOf": [
{
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
{
"type": "null"
}
]
}
},
"required": [
"liftPts",
"targetPct",
"perVersion",
"total",
"days"
],
"additionalProperties": false
},
"description": "The same test for B at +2, +5, +10 and +15 points; null where B would pass 100%"
},
"summary": {
"type": "string"
},
"permalink": {
"type": "string",
"description": "A page on this site that shows this result, for a person to open"
}
},
"required": [
"perVersion",
"total",
"days",
"baselinePct",
"targetPct",
"alpha",
"power",
"method",
"otherLifts",
"summary",
"permalink"
],
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": false
}🟢judge_agreement(rows, csv)
Compares a person's verdicts with an AI judge's verdicts on the same results. Returns the agreement rate, Cohen's kappa with confidence intervals, precision and recall per verdict, the agreement of a judge that always gives the most common verdict, a trust verdict (kappa 0.6 or more with at least 80% recall on every verdict; 0.4 to 0.6 is borderline; under 30 results is too few) and the rows where the two disagree.
Eingabe-Schema
{
"type": "object",
"properties": {
"rows": {
"description": "One row per result. Common words are mapped onto two verdicts: worked (worked, pass, yes, true, 1, good, kept, accepted, ok) and didn't work (didn't work, fail, no, false, 0, bad, redo, retry, rejected, wrong). Other labels are compared as written.",
"maxItems": 5000,
"type": "array",
"items": {
"type": "object",
"properties": {
"human": {
"type": "string",
"minLength": 1,
"maxLength": 100,
"description": "The person's verdict on one result"
},
"judge": {
"type": "string",
"minLength": 1,
"maxLength": 100,
"description": "The AI judge's verdict on the same result"
},
"note": {
"description": "What the result was, to recognise it in the disagreements",
"type": "string",
"maxLength": 300
}
},
"required": [
"human",
"judge"
]
}
},
"csv": {
"description": "The same data as CSV or TSV text instead of rows: columns named human and judge (and optionally note), or the first two columns in that order.",
"type": "string",
"maxLength": 500000
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema"
}Ausgabe-Schema
{
"type": "object",
"properties": {
"n": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991,
"description": "Results compared"
},
"skippedRows": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991,
"description": "CSV lines without both verdicts"
},
"agreement": {
"type": "number",
"description": "Share of results where the judge and the person gave the same verdict"
},
"agreementCi": {
"type": "array",
"prefixItems": [
{
"type": "number"
},
{
"type": "number"
}
],
"items": false,
"minItems": 2,
"maxItems": 2,
"description": "95% confidence interval"
},
"kappa": {
"description": "Cohen's kappa: agreement beyond chance. Null when it is undefined.",
"type": [
"number",
"null"
]
},
"kappaCi": {
"anyOf": [
{
"type": "array",
"prefixItems": [
{
"type": "number"
},
{
"type": "number"
}
],
"items": false,
"minItems": 2,
"maxItems": 2
},
{
"type": "null"
}
]
},
"kappaBand": {
"type": "string",
"description": "Landis and Koch label for kappa"
},
"alwaysSameVerdict": {
"anyOf": [
{
"type": "object",
"properties": {
"label": {
"type": "string"
},
"agreement": {
"type": "number",
"description": "0 to 1"
}
},
"required": [
"label",
"agreement"
],
"additionalProperties": false
},
{
"type": "null"
}
],
"description": "The agreement a judge would get by always giving the most common human verdict"
},
"verdict": {
"type": "object",
"properties": {
"level": {
"type": "string",
"enum": [
"too-few",
"trust",
"borderline",
"dont-trust"
]
},
"headline": {
"type": "string"
},
"detail": {
"type": "string"
}
},
"required": [
"level",
"headline",
"detail"
],
"additionalProperties": false
},
"warnings": {
"type": "array",
"items": {
"type": "string"
},
"description": "Problems with the labelled sample itself"
},
"perVerdict": {
"type": "array",
"items": {
"type": "object",
"properties": {
"label": {
"type": "string"
},
"precision": {
"type": "number",
"description": "Of the results the judge gave this verdict, share the person agreed with"
},
"recall": {
"type": "number",
"description": "Of the results the person gave this verdict, share the judge also gave"
},
"support": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991,
"description": "Results the person gave this verdict"
},
"judged": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991,
"description": "Results the judge gave this verdict"
}
},
"required": [
"label",
"precision",
"recall",
"support",
"judged"
],
"additionalProperties": false
}
},
"matrix": {
"type": "object",
"properties": {
"labels": {
"type": "array",
"items": {
"type": "string"
}
},
"counts": {
"type": "array",
"items": {
"type": "array",
"items": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
}
},
"description": "counts[person][judge]"
}
},
"required": [
"labels",
"counts"
],
"additionalProperties": false
},
"disagreementCount": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"disagreements": {
"type": "array",
"items": {
"type": "object",
"properties": {
"row": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991,
"description": "1-based position in the rows sent"
},
"human": {
"type": "string"
},
"judge": {
"type": "string"
},
"note": {
"type": [
"string",
"null"
]
}
},
"required": [
"row",
"human",
"judge",
"note"
],
"additionalProperties": false
},
"description": "The first 25 results where the verdicts differ"
},
"summary": {
"type": "string"
},
"permalink": {
"type": "string",
"description": "A page on this site that shows this result, for a person to open"
}
},
"required": [
"n",
"skippedRows",
"agreement",
"agreementCi",
"kappa",
"kappaCi",
"kappaBand",
"alwaysSameVerdict",
"verdict",
"warnings",
"perVerdict",
"matrix",
"disagreementCount",
"disagreements",
"summary",
"permalink"
],
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": false
}🟢build_judge_prompt(block, featureDescription, criteria, workedExamples, failedExamples, ...)
Builds a pass/fail LLM-as-a-judge prompt for one kind of AI step (make, answer, transform, extract, judge, find, predict, plan, act or learn). Returns the prompt with {{input}} and {{result}} placeholders, numbered pass conditions (the block's starter conditions unless you send your own), up to 3 pass and 3 fail examples interleaved, and a JSON or XML answer format.
Eingabe-Schema
{
"type": "object",
"properties": {
"block": {
"type": "string",
"enum": [
"make",
"answer",
"transform",
"extract",
"judge",
"find",
"predict",
"plan",
"act",
"learn"
],
"description": "What the AI step does. make: creates new content from a request (an image, a post, slides, advice); answer: replies to a question that has a right answer (a support reply, a policy question, a how-to); transform: reshapes material faithfully (a rewrite, a translation, text to speech); extract: pulls out what's in the input (fields from a PDF, speech to text, text from a photo); judge: gives a label, score or choice that can be checked now (moderation, routing, a guardrail); find: picks the few relevant items out of many (search, recommendations); predict: makes an estimate that can only be checked later (will this lead buy, is this fraud, when will it arrive); plan: turns a goal into steps (a research plan, a multi-step task); act: takes one action with side effects (send an email, take a payment, call an API); learn: updates a model or prompt from examples (a voice clone, prompt tuning)."
},
"featureDescription": {
"default": "",
"description": "Who uses the feature, what they ask for and what a good result gives them",
"type": "string",
"maxLength": 2000
},
"criteria": {
"description": "Pass conditions, one per item. Omitted: the starter conditions for the block.",
"minItems": 1,
"maxItems": 10,
"type": "array",
"items": {
"type": "string",
"minLength": 1,
"maxLength": 500
}
},
"workedExamples": {
"default": [],
"description": "Real results that worked first time; labelled pass in the prompt",
"maxItems": 3,
"type": "array",
"items": {
"type": "object",
"properties": {
"input": {
"default": "",
"description": "What the step received",
"type": "string",
"maxLength": 4000
},
"result": {
"type": "string",
"maxLength": 4000,
"description": "What the step produced, in full"
}
},
"required": [
"result"
]
}
},
"failedExamples": {
"default": [],
"description": "Real results that didn't work; labelled fail in the prompt",
"maxItems": 3,
"type": "array",
"items": {
"type": "object",
"properties": {
"input": {
"default": "",
"description": "What the step received",
"type": "string",
"maxLength": 4000
},
"result": {
"type": "string",
"maxLength": 4000,
"description": "What the step produced, in full"
}
},
"required": [
"result"
]
}
},
"outputFormat": {
"default": "json",
"description": "How the judge answers",
"type": "string",
"enum": [
"json",
"xml-tags"
]
},
"includeReasoning": {
"default": true,
"description": "Whether the judge writes a short reason before its verdict",
"type": "boolean"
}
},
"required": [
"block"
],
"$schema": "https://json-schema.org/draft/2020-12/schema"
}Ausgabe-Schema
{
"type": "object",
"properties": {
"prompt": {
"type": "string",
"description": "The judge prompt. Fill {{input}} and {{result}} for each result you check."
},
"block": {
"type": "string"
},
"criteria": {
"type": "array",
"items": {
"type": "string"
},
"description": "The pass conditions used"
},
"placeholders": {
"type": "array",
"items": {
"type": "string"
}
},
"unfilled": {
"type": "object",
"properties": {
"feature": {
"type": "boolean"
},
"criteria": {
"type": "boolean"
},
"workedExamples": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"failedExamples": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
}
},
"required": [
"feature",
"criteria",
"workedExamples",
"failedExamples"
],
"additionalProperties": false,
"description": "Parts of the prompt still shown as [bracketed] placeholders"
},
"leftOutOfPermalink": {
"type": "array",
"items": {
"type": "string"
},
"description": "Inputs too long for the permalink, so its page shows the prompt without them"
},
"summary": {
"type": "string"
},
"permalink": {
"type": "string",
"description": "A page on this site that shows this result, for a person to open"
}
},
"required": [
"prompt",
"block",
"criteria",
"placeholders",
"unfilled",
"leftOutOfPermalink",
"summary",
"permalink"
],
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": false
}🟢launch_checklist(done, group)
The 20 AI-specific launch checks for an AI app (spending caps, rate limits, server-side keys, credit charging, version tags, records of which results worked first time, real cost per result, switching models safely). Returns the checks not yet done, each with why it matters and a request for a coding agent, plus one combined request.
Eingabe-Schema
{
"type": "object",
"properties": {
"done": {
"default": [],
"description": "Ids of checks already done. Ids: spend-cap, rate-limits, server-keys, capped-retries, provider-budget, every-call-charged, deduct-server-side, refund-failures, one-charge-per-request, output-metadata, versioned-regenerations, outcome-events, events-by-version, step-costs, wasted-spend, who-it-fails-for, prompts-in-git, switch-flags, replay-set, shutdown-plan.",
"maxItems": 30,
"type": "array",
"items": {
"type": "string",
"maxLength": 60
}
},
"group": {
"description": "Only checks in one group: cost (cost and abuse), charging (charging for ai use), traceability (a record of every ai result), signals (what worked, for whom, at what cost), change (switching safely)",
"type": "string",
"enum": [
"cost",
"charging",
"traceability",
"signals",
"change"
]
}
},
"$schema": "https://json-schema.org/draft/2020-12/schema"
}Ausgabe-Schema
{
"type": "object",
"properties": {
"total": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"doneCount": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"groups": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"title": {
"type": "string"
},
"done": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
},
"total": {
"type": "integer",
"minimum": -9007199254740991,
"maximum": 9007199254740991
}
},
"required": [
"id",
"title",
"done",
"total"
],
"additionalProperties": false
}
},
"remaining": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"group": {
"type": "string"
},
"title": {
"type": "string"
},
"why": {
"type": "string"
},
"prompt": {
"type": "string",
"description": "A request for a coding agent that adds this check"
}
},
"required": [
"id",
"group",
"title",
"why",
"prompt"
],
"additionalProperties": false
}
},
"combinedPrompt": {
"type": "string",
"description": "One message covering every remaining check, numbered; empty when none remain"
},
"unknownIds": {
"type": "array",
"items": {
"type": "string"
},
"description": "Ids in done that aren't checks"
},
"summary": {
"type": "string"
},
"permalink": {
"type": "string",
"description": "A page on this site that shows this result, for a person to open"
}
},
"required": [
"total",
"doneCount",
"groups",
"remaining",
"combinedPrompt",
"unknownIds",
"summary",
"permalink"
],
"$schema": "https://json-schema.org/draft/2020-12/schema",
"additionalProperties": false
}Community
Nachweis