From c4942d873e7d9e21e2917bc81e6bcfb601fefffa Mon Sep 17 00:00:00 2001 From: Dante Date: Tue, 27 Oct 2026 03:35:11 +0800 Subject: [PATCH] docs: add public feedback goal fidelity evaluation --- docs/architecture/evaluator-rag-prototype.md | 4 ++ .../candidate-playbook.md | 47 ++++++++++++++++++ .../public-feedback-goal-fidelity/report.json | 23 +++++++++ .../scenario.json | 48 +++++++++++++++++++ .../public-feedback-goal-fidelity/trace.json | 33 +++++++++++++ .../verifier-result.json | 29 +++++++++++ 6 files changed, 184 insertions(+) create mode 100644 examples/evaluator-rag-prototype/public-feedback-goal-fidelity/candidate-playbook.md create mode 100644 examples/evaluator-rag-prototype/public-feedback-goal-fidelity/report.json create mode 100644 examples/evaluator-rag-prototype/public-feedback-goal-fidelity/scenario.json create mode 100644 examples/evaluator-rag-prototype/public-feedback-goal-fidelity/trace.json create mode 100644 examples/evaluator-rag-prototype/public-feedback-goal-fidelity/verifier-result.json diff --git a/docs/architecture/evaluator-rag-prototype.md b/docs/architecture/evaluator-rag-prototype.md index cb442ae0f..6f38852a5 100644 --- a/docs/architecture/evaluator-rag-prototype.md +++ b/docs/architecture/evaluator-rag-prototype.md @@ -123,6 +123,10 @@ Current corpus: expected-output comparisons, representative repository/commit histories, and regression commands before deep-analysis behavior can be promoted. +- `public-feedback-goal-fidelity`: compares a shortened draft with the chosen + outcome and input/process/result chain, preserving verification limits and + separating optional alternatives. Decisions are illustrative. + ## ECC Tools Mapping ECC Tools already flags missing RAG/evaluator evidence for retrieval, diff --git a/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/candidate-playbook.md b/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/candidate-playbook.md new file mode 100644 index 000000000..9dd7b3452 --- /dev/null +++ b/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/candidate-playbook.md @@ -0,0 +1,47 @@ +# Public Feedback Goal Fidelity Playbook + +Candidate id: `preserved-outcome-and-chain` + +This is a synthetic, read-only comparison, with expected rubric decisions rather +than a historical reproduction or measured model result. It responds to +[reported feedback](https://github.com/affaan-m/ECC/issues/3695). + +## Compare Meaning Before Shortening + +1. Extract the user's chosen outcome and important causal steps. +2. Shorten wording while retaining `INPUTS -> PROCESS -> RESULT` and why the + result matters. Here that purpose is sustained throughput. +3. Compare the result with the original task. A human-assisted helper can be an + explicitly offered alternative, but must not silently replace automation. +4. Retain uncertainty: a requested capability is not an implemented architecture; + technical limits and required verification still apply. + +The accepted and rejected drafts in `scenario.json` deliberately use the same +topic. The rejected draft changes the outcome even though it sounds helpful. +Evaluate meaning and connections, not just shared keywords or brevity. + +## Existing Artifact Control + +For an HTML-editing variant, provide the original artifact as evaluation input +before asserting preservation. The contract can require supported technical +corrections and removal of outdated emphasis while preserving useful facts, +article purpose, substantial narrative, section structure and image references. +Return copyable HTML when that is requested; do not promote the narrow correction +to a new article theme. Supported additions and removals remain allowed. + +If the original is unavailable, preservation remains unverified. A revised code +block establishes format only. Version comparison, rendering, factual checking +and user acceptance are distinct evidence states. Do not invent the missing +original or claim those checks passed. + +The source reports a corrected public comment, but its earlier revision was not +independently recovered. For the HTML follow-up, original HTML and the earlier +response remain unavailable; revised output was reported provided, with quality +unverified. Exact ECC activation, version and causal contribution are unknown. + +## Repository Validation + +Run `node tests/docs/evaluator-rag-prototype.test.js`, Markdown lint on touched +docs, and `git diff --check`. These repository checks do not validate a missing +historical HTML artifact or establish user acceptance. No publication occurs in +this evaluator example. diff --git a/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/report.json b/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/report.json new file mode 100644 index 000000000..c60019551 --- /dev/null +++ b/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/report.json @@ -0,0 +1,23 @@ +{ + "schema_version": "ecc.evaluator-rag.report.v1", + "scenario_id": "public-feedback-goal-fidelity", + "run_id": "illustrative-goal-fidelity", + "read_only": true, + "result": "illustrative_expected_result", + "findings": [ + { + "id": "meaning-before-length", + "severity": "warning", + "summary": "A shorter draft can retain vocabulary while losing the requested outcome or causal connection." + }, + { + "id": "format-not-quality", + "severity": "info", + "summary": "A revised HTML code block establishes output format, not verified preservation, rendering or factual correctness." + } + ], + "recommended_next_action": { + "candidate_id": "preserved-outcome-and-chain", + "action": "Compare meaning and limits with the task before publishing; offer alternatives explicitly." + } +} diff --git a/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/scenario.json b/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/scenario.json new file mode 100644 index 000000000..dbe850a79 --- /dev/null +++ b/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/scenario.json @@ -0,0 +1,48 @@ +{ + "schema_version": "ecc.evaluator-rag.scenario.v1", + "scenario_id": "public-feedback-goal-fidelity", + "title": "Preserve the chosen outcome and causal chain when shortening feedback", + "mode": "read_only_prototype", + "objective": "Compare a shortened draft with the requested outcome, causal steps and verification limits before publication.", + "sources": [ + { + "kind": "public_feedback", + "url": "https://github.com/affaan-m/ECC/issues/3695", + "purpose": "Reported corrected goal drift and proposed evaluation" + }, + { + "kind": "public_comment", + "url": "https://github.com/evilbruce666/1c-odata-mcp/issues/56#issuecomment-6385767160", + "purpose": "Corrected public text cited by the reporter; not recovery of its earlier revision" + } + ], + "evidence_limits": "The earlier published revision was not independently recovered. Original HTML and earlier HTML response are unavailable; revised output was reported supplied but not compared, rendered, fact-checked or accepted by the user. ECC activation/version and causal contribution remain unknown.", + "inputs": { + "chosen_outcome": "OUTCOME_A: an end-to-end automated workflow producing the prepared result", + "causal_chain": [ + "INPUTS: incoming data sources", + "PROCESS: intermediary tool layer prepares the result", + "RESULT: final prepared result, with sustained throughput as the purpose" + ], + "constraint": "Describe the requested capability, not a claim that it is implemented. Preserve genuine feasibility and verification limits.", + "accepted_draft": "Request an end-to-end flow from incoming sources through the intermediary tool layer to a prepared result, supporting sustained throughput. Implementation and feasibility remain to be verified.", + "rejected_draft": "Request a convenient helper that prepares suggestions and leaves routine exceptions for a person." + }, + "retrieval_questions": [ + "Does the draft preserve OUTCOME_A?", + "Are the inputs, process, result and throughput rationale still connected?", + "Are optional alternatives labeled as alternatives?", + "Are implementation and verification limits retained?" + ], + "forbidden_actions": [ + "publishing feedback from this read-only evaluator", + "silently replacing OUTCOME_A with human-assisted OUTCOME_B", + "claiming implementation or verified HTML correction from a revised code block alone" + ], + "acceptance_gates": [ + "chosen outcome remains unchanged", + "important causal steps and throughput rationale remain explicit", + "alternatives are separate proposals, not substituted requirements", + "unknown feasibility and verification remain explicit" + ] +} diff --git a/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/trace.json b/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/trace.json new file mode 100644 index 000000000..1487612d2 --- /dev/null +++ b/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/trace.json @@ -0,0 +1,33 @@ +{ + "schema_version": "ecc.evaluator-rag.trace.v1", + "scenario_id": "public-feedback-goal-fidelity", + "run_id": "illustrative-goal-fidelity", + "read_only": true, + "illustrative": true, + "events": [ + { + "phase": "observation", + "summary": "The synthetic task requests shorter public feedback about an automated end-to-end workflow." + }, + { + "phase": "retrieval", + "summary": "Read the supplied outcome, causal chain and limits before comparing drafts." + }, + { + "phase": "proposal", + "candidate_ids": [ + "preserved-outcome-and-chain", + "silent-human-assisted-substitution" + ] + }, + { + "phase": "verification", + "summary": "Expected rubric: retain automation, causal chain and throughput purpose; reject a convenient helper substituted for the requested outcome." + }, + { + "phase": "promotion", + "promoted_candidate_id": "preserved-outcome-and-chain", + "summary": "Illustrative content recommendation only; no public comment is created or edited." + } + ] +} diff --git a/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/verifier-result.json b/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/verifier-result.json new file mode 100644 index 000000000..a9b03355f --- /dev/null +++ b/examples/evaluator-rag-prototype/public-feedback-goal-fidelity/verifier-result.json @@ -0,0 +1,29 @@ +{ + "schema_version": "ecc.evaluator-rag.verifier.v1", + "scenario_id": "public-feedback-goal-fidelity", + "run_id": "illustrative-goal-fidelity", + "read_only": true, + "evaluation_status": "illustrative_expected_decisions_not_historical_reproduction", + "candidates": [ + { + "candidate_id": "preserved-outcome-and-chain", + "decision": "accepted", + "reasons": [ + "Preserves the end-to-end automated OUTCOME_A", + "Connects incoming sources through the intermediary tool layer to the prepared result", + "Preserves sustained throughput as the rationale", + "Does not claim the requested architecture is implemented or verified" + ] + }, + { + "candidate_id": "silent-human-assisted-substitution", + "decision": "rejected", + "reasons": [ + "Substitutes human-assisted OUTCOME_B for the requested outcome", + "Drops incoming sources and the sustained throughput rationale", + "Presents an alternative as if it were the original requirement" + ] + } + ], + "promoted_candidate_id": "preserved-outcome-and-chain" +}