schema_validator.py

script

← Back to skill

Content hash: 2929f00177a186c17aa32c3a52b61c1844cdb4c0ac7b314e90e69196f4ede802
#!/usr/bin/env python3
"""Validate LLM tool schemas against best practices before shipping them.

Checks the concrete rules from the tool-schema-design skill: naming, enums,
required-field minimalism, per-parameter descriptions, and overlap detection.
"""

from __future__ import annotations

from dataclasses import dataclass, field
from typing import Any


@dataclass
class ToolSchema:
    name: str
    description: str
    parameters: dict[str, Any]  # JSON Schema object
    required: list[str] = field(default_factory=list)


@dataclass
class Finding:
    tool: str
    severity: str  # "error" | "warn"
    message: str


def validate_tool(tool: ToolSchema) -> list[Finding]:
    findings: list[Finding] = []
    props = tool.parameters.get("properties", {})
    required = set(tool.required)

    # 1. Description is the primary signal - must exist and be specific
    if not tool.description or len(tool.description) < 20:
        findings.append(Finding(
            tool.name, "error",
            "Description too short (<20 chars). It's the model's ONLY signal.",
        ))

    # 2. Per-parameter descriptions
    for pname, pdef in props.items():
        if not pdef.get("description"):
            findings.append(Finding(
                tool.name, "warn", f"Parameter '{pname}' missing description.",
            ))

    # 3. Enum for finite-value params (looks for string params w/o enum)
    for pname, pdef in props.items():
        if pdef.get("type") == "string" and "enum" not in pdef:
            # Heuristic: params named like a mode/status/priority likely finite
            if any(k in pname.lower() for k in ("mode", "status", "priority", "type", "level")):
                findings.append(Finding(
                    tool.name, "warn",
                    f"'{pname}' looks finite-valued but has no 'enum' constraint.",
                ))

    # 4. Required minimalism - every required field is a failure point
    if len(required) > 3:
        findings.append(Finding(
            tool.name, "warn",
            f"{len(required)} required params. Can any be deferred/defaulted?",
        ))

    # 5. Unit/format hints for numeric/date params
    for pname, pdef in props.items():
        t = pdef.get("type")
        if t in ("number", "integer") and not any(
            k in pdef.get("description", "").lower() for k in ("ms", "sec", "bytes", "percent", "%", "unit")
        ):
            findings.append(Finding(
                tool.name, "warn",
                f"Numeric param '{pname}' lacks a unit hint in its description.",
            ))

    return findings


def detect_overlap(tools: list[ToolSchema]) -> list[Finding]:
    """Flag pairs of tools whose names/descriptions are too similar."""
    findings: list[Finding] = []
    for i in range(len(tools)):
        for j in range(i + 1, len(tools)):
            a, b = tools[i], tools[j]
            shared = set(a.description.lower().split()) & set(b.description.lower().split())
            name_similar = a.name.lower() in b.name.lower() or b.name.lower() in a.name.lower()
            if name_similar and len(shared) > 4:
                findings.append(Finding(
                    f"{a.name} / {b.name}", "warn",
                    "Overlapping tools - model will confuse them. Add boundary notes to descriptions.",
                ))
    return findings


# ── Demo ───────────────────────────────────────────────────────────────────

GOOD_TOOL = ToolSchema(
    name="get_calendar_events",
    description="Retrieve calendar events for a date or range. Returns start/end times and titles.",
    parameters={
        "type": "object",
        "properties": {
            "start_date": {"type": "string", "description": "ISO 8601 date (YYYY-MM-DD)"},
            "end_date": {"type": "string", "description": "ISO 8601 date (YYYY-MM-DD), optional"},
        },
    },
    required=["start_date"],
)

BAD_TOOL = ToolSchema(
    name="getData",
    description="Gets data.",
    parameters={
        "type": "object",
        "properties": {
            "mode": {"type": "string"},           # no enum, no description
            "uid": {"type": "integer"},           # no description, no unit
            "x": {"type": "string"},              # no description
            "y": {"type": "string"},              # no description
            "z": {"type": "string"},              # no description
        },
    },
    required=["mode", "uid", "x", "y", "z"],      # 5 required = 5 failure points
)


def main() -> None:
    print("=== Validating: GOOD tool ===")
    for f in validate_tool(GOOD_TOOL):
        print(f"  [{f.severity}] {f.message}")

    print("\n=== Validating: BAD tool ===")
    for f in validate_tool(BAD_TOOL):
        print(f"  [{f.severity}] {f.message}")

    print("\n=== Overlap detection ===")
    overlapping = [
        ToolSchema(
            name="search_products",
            description="Search products by keyword and return matching product listings with prices.",
            parameters={"type": "object", "properties": {}},
        ),
        ToolSchema(
            name="search_products_by_price",
            description="Search products by keyword and return matching product listings sorted by price.",
            parameters={"type": "object", "properties": {}},
        ),
    ]
    for f in detect_overlap(overlapping):
        print(f"  [{f.severity}] {f.message}")


if __name__ == "__main__":
    main()