{
  "id": "benchmark_evaluation_harness",
  "label": "Benchmark & Evaluation Harness Canonical",
  "phase": "phase-03",
  "category": "cognitive_flow",
  "status": "ACTIVE_DH_MIRROR",
  "purpose": "Define how DEVON-DEV measures model, runtime, retrieval, prompt, planning, validation and policy behavior before selecting or promoting any local AI component.",
  "function": "Turn runtime, model, server and cognitive-flow selection into evidence by running benchmark suites for error reading, cause location, clean REPLACE patch generation, APPEND blocking, policy compliance, evidence use, FSM respect, external documentation interpretation, sandbox planning and production-safe validation.",
  "benchmark_contract": {
    "requires_error_reading_benchmark": true,
    "requires_cause_location_benchmark": true,
    "requires_replace_patch_benchmark": true,
    "requires_append_guard_benchmark": true,
    "requires_policy_following_benchmark": true,
    "requires_evidence_use_benchmark": true,
    "requires_no_decision_by_llm_benchmark": true,
    "requires_fsm_respect_benchmark": true,
    "requires_external_documentation_interpretation_benchmark": true,
    "requires_sandbox_plan_benchmark": true,
    "requires_production_safe_validation_benchmark": true,
    "requires_baseline_registry": true,
    "requires_regression_history": true,
    "requires_promotion_decision_link": true,
    "decision_authority": "NONE_FSM_DECIDES"
  },
  "technology_requirements": [
    {
      "technology": "Benchmark Runner",
      "required_for": "Run DEVON-DEV benchmark suites and record scores for model, runtime, prompt, retrieval, planning and validation behavior.",
      "required_by_phase": "phase-03",
      "required_by_category": "benchmark_evaluation_harness",
      "required_by_buckets": [
        "Prerequisites",
        "Installation",
        "Configuration",
        "Validation",
        "Observable Evidence",
        "Failure Modes & Recovery",
        "Completion & Promotion"
      ],
      "expected_version": "MISSING until benchmark runner runtime, suites and baseline registry exist",
      "validation_command": "MISSING until benchmark runner can execute suites and emit score, artifact, baseline and regression records",
      "evidence_path": "/home/yeff/public_html/devon/canon/execution/state/dh_sources/benchmark_evaluation_harness_canonical.json",
      "status_source": "Documentation Hub + future Benchmark Runner + ModelOps / LLMOps Registry + Evidence Store",
      "blocking_if_missing": true,
      "failure_impact": "Without Benchmark Runner, DEVON-DEV chooses model, runtime, prompt behavior and server strategy by opinion instead of measured evidence."
    },
    {
      "technology": "Baseline Registry",
      "required_for": "Store benchmark baselines, model/runtime comparisons, regression thresholds and promotion references.",
      "required_by_phase": "phase-03",
      "required_by_category": "benchmark_evaluation_harness",
      "required_by_buckets": [
        "Configuration",
        "Validation",
        "Observable Evidence",
        "Failure Modes & Recovery",
        "Completion & Promotion"
      ],
      "expected_version": "MISSING until baseline schema and historical benchmark storage exist",
      "validation_command": "MISSING until baseline creation, lookup, comparison and regression-threshold checks exist",
      "evidence_path": "/home/yeff/public_html/devon/canon/execution/state/dh_sources/benchmark_evaluation_harness_canonical.json",
      "status_source": "Documentation Hub + future Baseline Registry + ModelOps / LLMOps Registry",
      "blocking_if_missing": true,
      "failure_impact": "Without Baseline Registry, DEVON-DEV cannot prove whether a model, prompt, runtime or retrieval change improved or degraded capability."
    },
    {
      "technology": "Regression Evaluation Suite",
      "required_for": "Detect behavior degradation across model versions, runtime changes, prompt changes, retrieval changes and evidence policies.",
      "required_by_phase": "phase-03",
      "required_by_category": "benchmark_evaluation_harness",
      "required_by_buckets": [
        "Validation",
        "Observable Evidence",
        "Failure Modes & Recovery",
        "Completion & Promotion"
      ],
      "expected_version": "MISSING until regression cases and pass/fail thresholds exist",
      "validation_command": "MISSING until regression suite can compare current run against baseline and block promotion on degradation",
      "evidence_path": "/home/yeff/public_html/devon/canon/execution/state/dh_sources/benchmark_evaluation_harness_canonical.json",
      "status_source": "Documentation Hub + future Benchmark Runner + ModelOps / LLMOps Registry",
      "blocking_if_missing": true,
      "failure_impact": "Without regression evaluation, DEVON-DEV can promote a model or prompt that passes a smoke test but loses core behavior under real engineering tasks."
    }
  ],
  "depends_on": [
    "cfc",
    "local_llm_runtime",
    "qwen_coder_model",
    "qwen_data_interpreter",
    "embeddings_engine",
    "vector_index",
    "external_knowledge_access",
    "source_trust_registry",
    "evidence_store",
    "knowledge_conflict_resolver",
    "prompt_governance",
    "prompt_instruction_assembly",
    "planning_engine",
    "impact_analysis_engine",
    "attention_focus_manager",
    "goals_manager",
    "modelops_llmops",
    "model_artifact_registry",
    "fsm_absolute_decision",
    "decision_rule_registry",
    "validation_matrix",
    "artifact_registry",
    "architecture_source_registry"
  ],
  "used_by": [
    "modelops_llmops",
    "model_artifact_registry",
    "local_llm_runtime",
    "qwen_coder_model",
    "qwen_data_interpreter",
    "embeddings_engine",
    "observability_core",
    "self_monitoring",
    "future_execution_monitoring_panel"
  ]
}