"""Deterministic scoring and rubric evaluation for WiseBench."""

import json
import re
from typing import Any, Optional




def check_required_sections(output: str, required_sections: list[str]) -> float:
    """Check if output contains all required sections/phrases.

    Scores 0.0 (none found) to 1.0 (all found).
    """
    if not output or not required_sections:
        return 1.0 if not required_sections else 0.0

    output_lower = output.lower() if isinstance(output, str) else ""
    found_count = sum(1 for section in required_sections if section.lower() in output_lower)
    return found_count / len(required_sections)


def check_forbidden_phrases(output: str, forbidden_phrases: list[str]) -> float:
    """Check that output does not contain forbidden phrases.

    Returns 1.0 (no forbidden) to 0.0 (one or more found).
    """
    if not output or not forbidden_phrases:
        return 1.0

    output_lower = output.lower() if isinstance(output, str) else ""
    for phrase in forbidden_phrases:
        if len(phrase) == 0:
            continue
        if phrase.lower() in output_lower:
            return 0.0

    return 1.0


def check_max_words(output: str, max_words: Optional[int]) -> float:
    """Check if output is within word limit.

    Returns 1.0 (within limit) to 0.0 (exceeded).
    """
    if max_words is None or not output:
        return 1.0

    # Count words
    words = len(output.split())
    ratio = min(words / max_words, 1.0)

    if words <= max_words:
        return 1.0
    elif words <= max_words * 1.5:
        # Partial penalty for slight overage
        return 0.7
    else:
        return 0.3


def check_valid_json(output: str, allow_partial: bool = True) -> float:
    """Check if output is valid JSON.

    Returns 1.0 (valid) to 0.0 (invalid).
    """
    if not output:
        return 0.0

    # Clean up common markdown wrappers
    cleaned = re.sub(r'^```json\s*', '', output.strip())
    cleaned = re.sub(r'\s*```$', '', cleaned)
    cleaned = cleaned.strip()

    try:
        parsed = json.loads(cleaned)
        if isinstance(parsed, (dict, list)):
            return 1.0
        return 0.0
    except json.JSONDecodeError:
        return 0.0


def check_must_include(output: str, must_include: list[str]) -> float:
    """Check if output contains all required words/phrases.

    Returns 0.0 (none found) to 1.0 (all found).
    """
    if not output or not must_include:
        return 1.0 if not must_include else 0.0

    output_lower = output.lower() if isinstance(output, str) else ""
    found_count = sum(1 for phrase in must_include if phrase.lower() in output_lower)
    return found_count / len(must_include)


def check_must_not_include(output: str, must_not_include: list[str]) -> float:
    """Check that output does not contain forbidden words/phrases.

    Returns 0.0 (one or more found) to 1.0 (none found).
    """
    if not output or not must_not_include:
        return 1.0

    output_lower = output.lower() if isinstance(output, str) else ""
    for phrase in must_not_include:
        if len(phrase) == 0:
            continue
        if phrase.lower() in output_lower:
            return 0.0

    return 1.0


def evaluate_deterministic_checks(task_id: str, output: str, deterministic_checks: dict) -> tuple[float, list[str]]:
    """Evaluate all deterministic checks for a task.

    Returns:
        (score, failed_requirements_list)
    """
    score = 1.0
    failed = []

    # Check each criterion
    checks = deterministic_checks or {}

    if "required_sections" in checks and checks["required_sections"]:
        section_score = check_required_sections(
            output, checks["required_sections"]
        )
        score *= section_score
        if not isinstance(section_score, float) or abs(1.0 - section_score) > 0.1:
            failed.append("missing_required_sections")

    if "forbidden_phrases" in checks and checks["forbidden_phrases"]:
        forbidden_score = check_forbidden_phrases(
            output, checks["forbidden_phrases"]
        )
        score *= forbidden_score
        if not isinstance(forbidden_score, float) or abs(1.0 - forbidden_score) > 0.1:
            failed.append("contains_forbidden_phrases")

    if "max_words" in checks and checks["max_words"]:
        word_score = check_max_words(output, checks["max_words"])
        score *= word_score
        if not isinstance(word_score, float) or abs(1.0 - word_score) > 0.001:
            failed.append("exceeds_word_limit")

    if "valid_json" in checks and checks["valid_json"]:
        json_score = check_valid_json(output)
        score *= json_score
        if not isinstance(json_score, float) or abs(1.0 - json_score) > 0.1:
            failed.append("invalid_json")

    if "must_include" in checks and checks["must_include"]:
        include_score = check_must_include(output, checks["must_include"])
        score *= include_score
        if not isinstance(include_score, float) or abs(1.0 - include_score) > 0.001:
            failed.append("missing_required_phrases")

    if "must_not_include" in checks and checks["must_not_include"]:
        exclude_score = check_must_not_include(output, checks["must_not_include"])
        score *= exclude_score
        if not isinstance(exclude_score, float) or abs(1.0 - exclude_score) > 0.5:
            failed.append("contains_forbidden_phrases")

    return max(0.0, score), failed


def load_scoring_profile(profile_id: str) -> Optional[dict]:
    """Load a scoring profile by ID."""
    from .tasks import get_scoring_profile
    return get_scoring_profile(profile_id)


def evaluate_rubric_score(output: str, profile: dict) -> float:
    """Evaluate output against rubric weights.

    This is a placeholder for when we implement actual quality scoring.
    For now, returns 0.5 as neutral score - the judge or detailed analysis
    would provide better scores.
    """
    if not output:
        return 0.0

    # Simple heuristic: longer outputs within reasonable bounds get higher scores
    word_count = len(output.split())
    if word_count < 50:
        return 0.3
    elif word_count < 200:
        return 0.6
    else:
        return 0.8


def calculate_deterministic_score(
    task_id: str, output: str, deterministic_checks: dict
) -> float:
    """Calculate the deterministic portion of a score."""
    if not output:
        return 0.0

    score, _ = evaluate_deterministic_checks(task_id, output, deterministic_checks)
    return score


def calculate_final_score(
    quality_score: float, instruction_score: float, reliability_score: float, speed_score: float
) -> float:
    """Calculate final score using default weighting.

    Formula: quality × 0.60 + instruction × 0.20 + reliability × 0.10 + speed × 0.10
    """
    return (
        quality_score * 0.60 +
        instruction_score * 0.20 +
        reliability_score * 0.10 +
        speed_score * 0.10
    )


def calculate_coding_final_score(
    unit_test_score: float, quality_score: float, instruction_score: float,
    format_score: float, speed_score: float
) -> float:
    """Calculate final score for coding tasks.

    Formula: unit_test × 0.50 + quality × 0.20 + instruction × 0.15 + format × 0.10 + speed × 0.05
    """
    return (
        unit_test_score * 0.50 +
        quality_score * 0.20 +
        instruction_score * 0.15 +
        format_score * 0.10 +
        speed_score * 0.05
    )


def calculate_tool_calling_final_score(
    tool_selection_score: float, valid_arguments_score: float, task_completion_score: float,
    error_recovery_score: float, wasted_steps_score: float
) -> float:
    """Calculate final score for tool calling tasks.

    Formula: correct_tool × 0.25 + valid_args × 0.25 + task_completed × 0.25 + error_recovery × 0.15 + minimal_wasted × 0.10
    """
    return (
        tool_selection_score * 0.25 +
        valid_arguments_score * 0.25 +
        task_completion_score * 0.25 +
        error_recovery_score * 0.15 +
        wasted_steps_score * 0.10
    )

import json
import re


def evaluate(task, output):
    checks = task.get('deterministic_checks') or {}
    active = {k: v for k, v in checks.items() if v and k in {
        'required_sections', 'forbidden_phrases', 'max_words', 'valid_json', 'must_include', 'must_not_include'}}
    schema = task.get('json_fields')
    if not active and not schema:
        return {'check_score': None, 'failed_requirements': [], 'check_results': [], 'quality_score': None,
                'note': 'No supported deterministic checks; quality not evaluated.'}
    score, failed = evaluate_deterministic_checks(task['id'], output, active)
    check_results = []
    check_functions = {
        'required_sections': check_required_sections,
        'forbidden_phrases': check_forbidden_phrases,
        'max_words': check_max_words,
        'valid_json': check_valid_json,
        'must_include': check_must_include,
        'must_not_include': check_must_not_include,
    }
    for name, expected in active.items():
        value = check_functions[name](output, expected)
        check_results.append({'check': name, 'passed': value == 1.0, 'score': value})
    if schema:
        try:
            value = json.loads(re.sub(r'\s*```$', '', re.sub(r'^```(?:json)?\s*', '', output.strip(), flags=re.I)).strip())
            types = {'string': str, 'number': (int, float), 'boolean': bool}
            invalid = [key for key, kind in schema.items() if not isinstance(value, dict) or key not in value or not isinstance(value[key], types[kind]) or (kind == 'number' and isinstance(value[key], bool))]
        except (ValueError, KeyError, TypeError):
            invalid = list(schema)
        if invalid:
            score = 0.0
            failed.append('invalid_json_fields: ' + ', '.join(invalid))
        check_results.append({'check': 'json_fields', 'passed': not invalid,
                              'score': 0.0 if invalid else 1.0,
                              'failed_fields': invalid})
    return {'check_score': score, 'failed_requirements': failed, 'check_results': check_results, 'quality_score': None,
            'note': 'Checks measure specified constraints, not overall answer quality.'}



TASKS=json.loads("[{\"id\":\"agentic_workflow_001\",\"category\":\"agentic_workflow\",\"title\":\"Multi-Step Research Workflow\",\"description\":\"Plan and execute a research task with multiple steps.\",\"difficulty\":\"hard\",\"scoring_profile\":\"agentic_workflow_v1\",\"tags\":[\"workflow\",\"research\"],\"system_prompt\":\"You are an AI agent. Plan and execute complex multi-step tasks systematically.\",\"user_prompt\":\"Plan a research workflow to compare Python vs JavaScript for web development:\\n1. Identify key comparison criteria\\n2. Explain pros of each language\\n3. Compare performance considerations\\n4. Provide final recommendation\\nPresent as structured steps with reasoning.\",\"expected_format\":{},\"deterministic_checks\":{\"must_include\":[\"Python\",\"JavaScript\",\"step\"],\"valid_json\":false,\"max_words\":300}},{\"id\":\"coding_001\",\"category\":\"coding\",\"title\":\"Python Fibonacci Generator\",\"description\":\"Create a Python function that generates Fibonacci numbers.\",\"difficulty\":\"easy\",\"scoring_profile\":\"coding_v1\",\"tags\":[\"python\",\"programming\"],\"system_prompt\":\"You are a senior software engineer. Write clean, well-documented code with proper error handling.\",\"user_prompt\":\"Write a Python function that generates the first n Fibonacci numbers and returns them as a list. The function should:\\n- Take an integer n as parameter\\n- Return empty list for n <= 0\\n- Handle edge cases properly\",\"expected_format\":{\"required_sections\":[\"def fib\"],\"forbidden_phrases\":[]},\"deterministic_checks\":{\"must_include\":[\"def\",\"fibonacci\",\"list\",\"range\"],\"valid_json\":false,\"max_words\":300}},{\"id\":\"coding_002\",\"category\":\"coding\",\"title\":\"JavaScript Array Sum\",\"description\":\"Create a JavaScript function to calculate sum of array elements.\",\"difficulty\":\"easy\",\"scoring_profile\":\"coding_v1\",\"tags\":[\"javascript\",\"programming\"],\"system_prompt\":\"You are a software engineer. Write clean, efficient code with proper type annotations where applicable.\",\"user_prompt\":\"Write a JavaScript function that takes an array of numbers and returns their sum. The function should:\\n- Handle empty arrays (return 0)\\n- Use reduce or a for loop\",\"expected_format\":{\"required_sections\":[\"function\",\"sum\"],\"forbidden_phrases\":[]},\"deterministic_checks\":{\"must_include\":[\"function\",\"array\",\"sum\"],\"valid_json\":false,\"max_words\":250}},{\"id\":\"coding_003\",\"category\":\"coding\",\"title\":\"Python JSON to Dictionary\",\"description\":\"Write a Python function to parse JSON string and convert to dict.\",\"difficulty\":\"easy\",\"scoring_profile\":\"coding_v1\",\"tags\":[\"python\",\"json\",\"data\"],\"system_prompt\":\"You are a Python developer. Write clean, well-structured code with proper error handling.\",\"user_prompt\":\"Write a Python function that:\\n- Takes a JSON string as input\\n- Parses it into a Python dictionary\\n- Returns None if the JSON is invalid\\n- Includes proper error handling\",\"expected_format\":{\"required_sections\":[\"import json\",\"json.loads\"],\"forbidden_phrases\":[]},\"deterministic_checks\":{\"must_include\":[\"json\",\"loads\",\"try\",\"except\"],\"valid_json\":false,\"max_words\":200}},{\"id\":\"general_chat_001\",\"category\":\"general_chat\",\"title\":\"Casual Conversation About Movies\",\"description\":\"Have a natural conversation about movie preferences.\",\"difficulty\":\"easy\",\"scoring_profile\":\"general_chat_v1\",\"tags\":[\"conversation\",\"entertainment\"],\"system_prompt\":\"You are a friendly conversational assistant. Engage in natural, engaging dialogue.\",\"user_prompt\":\"Let's chat about movies! I recently watched 'Inception' and loved it. What's a movie you'd recommend? Please respond with a friendly, conversational tone.\",\"expected_format\":{},\"deterministic_checks\":{\"must_include\":[\"movie\",\"recommend\"],\"valid_json\":false,\"max_words\":150}},{\"id\":\"general_chat_002\",\"category\":\"general_chat\",\"title\":\"Helpful Travel Advice\",\"description\":\"Provide travel recommendations with local knowledge.\",\"difficulty\":\"easy\",\"scoring_profile\":\"general_chat_v1\",\"tags\":[\"travel\",\"advice\"],\"system_prompt\":\"You are a helpful travel assistant. Provide practical, useful advice.\",\"user_prompt\":\"I'm planning a trip to Japan for my first time visiting. What are some must-see places in Tokyo? Keep it friendly and practical.\",\"expected_format\":{},\"deterministic_checks\":{\"must_include\":[\"Tokyo\",\"Japan\",\"place\"],\"valid_json\":false,\"max_words\":200}},{\"id\":\"information_001\",\"category\":\"information\",\"title\":\"Explain Quantum Computing Basics\",\"description\":\"Provide a clear explanation of quantum computing fundamentals.\",\"difficulty\":\"medium\",\"scoring_profile\":\"information_v1\",\"tags\":[\"technology\",\"education\"],\"system_prompt\":\"You are a helpful AI assistant. Provide accurate, clear explanations.\",\"user_prompt\":\"Explain what quantum computing is and how it differs from classical computing in simple terms. Include:\\n1. What qubits are\\n2. Superposition principle\\n3. Entanglement concept\\n4. Practical applications\",\"expected_format\":{\"required_sections\":[\"qubits\",\"superposition\",\"entanglement\",\"applications\"]},\"deterministic_checks\":{\"required_sections\":[\"qubit\",\"superposition\",\"entanglement\"],\"valid_json\":false,\"max_words\":500}},{\"id\":\"information_002\",\"category\":\"information\",\"title\":\"Explain Climate Change\",\"description\":\"Provide a comprehensive overview of climate change.\",\"difficulty\":\"medium\",\"scoring_profile\":\"information_v1\",\"tags\":[\"science\",\"environment\"],\"system_prompt\":\"You are an educational AI assistant. Provide factual, well-structured information.\",\"user_prompt\":\"Explain climate change and its main causes. Include:\\n1. What greenhouse gases are\\n2. Human activities causing warming\\n3. Impacts on ecosystems\\n4. Mitigation strategies\",\"expected_format\":{\"required_sections\":[\"greenhouse gases\",\"human activities\",\"impacts\",\"mitigation\"]},\"deterministic_checks\":{\"required_sections\":[\"greenhouse effect\",\"carbon dioxide\",\"temperature\"],\"valid_json\":false,\"max_words\":400}},{\"id\":\"information_003\",\"category\":\"information\",\"title\":\"Explain Blockchain Technology\",\"description\":\"Explain blockchain fundamentals and use cases.\",\"difficulty\":\"medium\",\"scoring_profile\":\"information_v1\",\"tags\":[\"technology\",\"finance\"],\"system_prompt\":\"You are a knowledgeable AI assistant explaining complex topics simply.\",\"user_prompt\":\"Explain what blockchain is and how it works. Include:\\n1. What blocks and chains mean\\n2. Decentralization concept\\n3. Cryptographic hashing\\n4. Real-world examples\",\"expected_format\":{\"required_sections\":[\"blocks\",\"decentralized\",\"hashing\",\"examples\"]},\"deterministic_checks\":{\"required_sections\":[\"blockchain\",\"decentralized\",\"cryptographic\"],\"valid_json\":false,\"max_words\":450}},{\"id\":\"instruction_following_001\",\"category\":\"instruction_following\",\"title\":\"Follow Complex Instructions\",\"description\":\"Test ability to follow multi-step instructions precisely.\",\"difficulty\":\"medium\",\"scoring_profile\":\"instruction_following_v1\",\"tags\":[\"following\",\"instructions\"],\"system_prompt\":\"You are an instruction-following AI. Follow all constraints exactly as stated.\",\"user_prompt\":\"Write a short paragraph about cats with these requirements:\\n- Exactly 3 sentences\\n- Each sentence must start with the letter 'C'\\n- Do not use the word 'paw' or 'meow'\\n- Keep it friendly and positive\",\"expected_format\":{\"required_sections\":[\"cat\"],\"forbidden_phrases\":[\"paw\",\"meow\"]},\"deterministic_checks\":{\"must_include\":[\"sentence\",\"three\",\"cats\"],\"valid_json\":false,\"max_words\":100}},{\"id\":\"instruction_following_002\",\"category\":\"instruction_following\",\"title\":\"Write Technical Documentation\",\"description\":\"Create API documentation following specific formatting rules.\",\"difficulty\":\"medium\",\"scoring_profile\":\"instruction_following_v1\",\"tags\":[\"documentation\",\"technical\"],\"system_prompt\":\"You are a technical writer. Create clear, well-formatted documentation.\",\"user_prompt\":\"Document the following function with these rules:\\n- Use markdown code blocks for examples\\n- Include parameter descriptions\\n- Add a return value section\\n- Write in present tense\",\"expected_format\":{},\"deterministic_checks\":{\"must_include\":[\"parameter\",\"return\",\"function\"],\"valid_json\":false,\"max_words\":200}},{\"id\":\"json_output_001\",\"category\":\"json_output\",\"title\":\"Generate User Object JSON\",\"description\":\"Create a properly formatted JSON object with user data.\",\"difficulty\":\"easy\",\"scoring_profile\":\"json_output_v1\",\"tags\":[\"json\",\"data\"],\"system_prompt\":\"You are a data specialist. Generate valid, well-structured JSON output.\",\"user_prompt\":\"Create a JSON object representing a user with these fields:\\n- id (number)\\n- name (string)\\n- email (string)\\n- is_active (boolean)\\n\\nMake sure the JSON is valid and properly formatted.\",\"expected_format\":{\"valid_json\":true},\"deterministic_checks\":{\"must_include\":[\"name\",\"email\"],\"valid_json\":true,\"max_words\":null},\"json_fields\":{\"id\":\"number\",\"name\":\"string\",\"email\":\"string\",\"is_active\":\"boolean\"}},{\"id\":\"json_output_002\",\"category\":\"json_output\",\"title\":\"Generate Product Array JSON\",\"description\":\"Create a JSON array of products with required fields.\",\"difficulty\":\"easy\",\"scoring_profile\":\"json_output_v1\",\"tags\":[\"json\",\"ecommerce\"],\"system_prompt\":\"You are an e-commerce specialist. Generate valid JSON product data.\",\"user_prompt\":\"Create a JSON array containing exactly 2 products. Each product should have:\\n- name (string)\\n- price (number)\\n- in_stock (boolean)\\n\\nEnsure the JSON is valid and properly formatted.\",\"expected_format\":{\"valid_json\":true},\"deterministic_checks\":{\"must_include\":[\"name\",\"price\"],\"valid_json\":true,\"max_words\":null}},{\"id\":\"marketing_copy_001\",\"category\":\"marketing_copy\",\"title\":\"Write Product Launch Email\",\"description\":\"Create an engaging email for product launch.\",\"difficulty\":\"medium\",\"scoring_profile\":\"marketing_copy_v1\",\"tags\":[\"marketing\",\"email\"],\"system_prompt\":\"You are a marketing copywriter. Write compelling, persuasive copy.\",\"user_prompt\":\"Write an email announcing the launch of a new AI productivity tool called 'FocusFlow'.\\nRequirements:\\n- Use engaging, upbeat tone\\n- Include call-to-action button text\\n- Mention key features: task management, AI scheduling, analytics\",\"expected_format\":{},\"deterministic_checks\":{\"must_include\":[\"FocusFlow\",\"launch\",\"productivity\"],\"valid_json\":false,\"max_words\":150}},{\"id\":\"reasoning_001\",\"category\":\"reasoning\",\"title\":\"Logical Deduction Puzzle\",\"description\":\"Solve a classic logic puzzle with step-by-step reasoning.\",\"difficulty\":\"medium\",\"scoring_profile\":\"reasoning_v1\",\"tags\":[\"logic\",\"puzzle\"],\"system_prompt\":\"You are a logical reasoning AI. Think step-by-step and provide clear explanations.\",\"user_prompt\":\"Three friends Alice, Bob, and Charlie are sitting in a row. \\n- Alice is not at either end\\n- Bob is to the right of Alice\\n- Charlie is at the rightmost position\\n\\nWho is in the middle? Explain your reasoning step by step.\",\"expected_format\":{},\"deterministic_checks\":{\"must_include\":[\"step\",\"middle\",\"Alice\",\"Bob\"],\"valid_json\":false,\"max_words\":300}},{\"id\":\"reasoning_002\",\"category\":\"reasoning\",\"title\":\"Time Calculation Problem\",\"description\":\"Calculate elapsed time with proper date/time reasoning.\",\"difficulty\":\"medium\",\"scoring_profile\":\"reasoning_v1\",\"tags\":[\"math\",\"time\"],\"system_prompt\":\"You are a helpful assistant. Solve problems carefully and show your work.\",\"user_prompt\":\"If a project starts at 9:45 AM on Monday and takes 3 days and 6 hours to complete, what time does it finish? Show your calculation step by step.\",\"expected_format\":{},\"deterministic_checks\":{\"must_include\":[\"step\",\"hours\",\"days\",\"time\"],\"valid_json\":false,\"max_words\":250}},{\"id\":\"reasoning_003\",\"category\":\"reasoning\",\"title\":\"Classification Problem\",\"description\":\"Categorize items based on given criteria.\",\"difficulty\":\"easy\",\"scoring_profile\":\"reasoning_v1\",\"tags\":[\"logic\",\"classification\"],\"system_prompt\":\"You are a classification AI. Apply the rules consistently and clearly.\",\"user_prompt\":\"Sort these fruits into categories:\\n- Apples, Bananas, Carrots, Dates, Eggs\\n\\nCategories: Fruits, Vegetables, Others\\nExplain which category each item belongs to and why.\",\"expected_format\":{},\"deterministic_checks\":{\"must_include\":[\"fruit\",\"vegetable\",\"carrot\",\"egg\"],\"valid_json\":false,\"max_words\":200}},{\"id\":\"research_quality_001\",\"category\":\"research_quality\",\"title\":\"Research Summary on Renewable Energy\",\"description\":\"Summarize key points about renewable energy trends.\",\"difficulty\":\"medium\",\"scoring_profile\":\"research_quality_v1\",\"tags\":[\"research\",\"energy\"],\"system_prompt\":\"You are a research analyst. Provide well-reasoned summaries with supporting details.\",\"user_prompt\":\"Summarize the current state of renewable energy adoption worldwide. Include:\\n- Growth trends in solar and wind\\n- Key challenges facing adoption\\n- Role of government policy\",\"expected_format\":{},\"deterministic_checks\":{\"must_include\":[\"solar\",\"wind\",\"renewable\"],\"valid_json\":false,\"max_words\":250}}]")
SUITE_ID="a1eb2e570f98d9b1002eec6a607661f0406919c6fb0e867fe14447d2d81e4610"
"""Downloadable community runner. The build bundles the shared scorer and tasks."""
import argparse
import json
import os
import sys
import time
import uuid
from pathlib import Path
from urllib.request import Request, urlopen


def request(url, data=None, headers=None):
    body = json.dumps(data).encode() if data is not None else None
    req = Request(url, body, {"Content-Type": "application/json", **(headers or {})})
    with urlopen(req, timeout=180) as response:
        return json.load(response)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--site", required=True)
    parser.add_argument("--endpoint", default="http://localhost:1234/v1")
    parser.add_argument("--model")
    parser.add_argument("--contributor", default="Anonymous")
    parser.add_argument("--retry", type=Path)
    args = parser.parse_args()
    if args.retry:
        report = json.loads(args.retry.read_text(encoding="utf-8"))
    else:
        endpoint = args.endpoint.rstrip("/")
        key = os.getenv("WISEBENCH_MODEL_KEY")
        headers = {"Authorization": "Bearer " + key} if key else {}
        model = args.model
        if not model:
            choices = [item["id"] for item in request(endpoint + "/models", headers=headers)["data"]]
            if len(choices) != 1:
                parser.error("Select a model with --model. Available: " + ", ".join(choices))
            model = choices[0]
        report = {"id": str(uuid.uuid4()), "suite": SUITE_ID, "model": model,
                  "contributor": args.contributor, "results": []}
        for task in TASKS:
            start = time.monotonic()
            row = {"task": task["id"], "score": None, "failed": True, "tokens": 0}
            try:
                reply = request(endpoint + "/chat/completions", {
                    "model": model, "messages": [{"role": "system", "content": task["system_prompt"]},
                    {"role": "user", "content": task["user_prompt"]}], "temperature": 0.2,
                    "top_p": 0.9, "max_tokens": 2000, "stream": False}, headers)
                output = reply["choices"][0]["message"]["content"]
                if not isinstance(output, str) or not output.strip():
                    raise ValueError("Empty response")
                row.update(score=evaluate(task, output)["check_score"], failed=False,
                           tokens=(reply.get("usage") or {}).get("completion_tokens") or 0)
            except Exception as error:
                print(task["id"], type(error).__name__, file=sys.stderr)
            row["ms"] = round((time.monotonic() - start) * 1000)
            report["results"].append(row)
            print(task["id"], "Failed" if row["failed"] else row["score"], flush=True)
        path = Path("wisebench-" + report["id"] + ".json")
        path.write_text(json.dumps(report, indent=2), encoding="utf-8")
        print("Saved", path, "· retry an upload with --retry", path)
    print(request(args.site.rstrip("/") + "/api/results", report))


if __name__ == "__main__":
    main()