File size: 5,193 Bytes
c4ae742 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 | export interface JudgeFeedbackSummary {
total_score?: number;
max_score?: number;
has_zeros?: boolean;
verdict?: string;
rationale?: string;
summary?: string;
dimensions?: Record<string, { score?: number; explanation?: string }>;
}
export interface IterationMemoryFinding {
dimension: string;
score: number | null;
message: string;
firstSeenIteration: number;
lastSeenIteration: number;
hits: number;
}
export interface IterationMemory {
updatedThroughIteration: number;
bestScore: number;
latestScore: number;
latestMaxScore: number;
latestVerdict: string;
activeFindings: IterationMemoryFinding[];
resolvedDimensions: string[];
latestFeedback: string;
}
const MAX_FINDINGS = 8;
const MAX_MESSAGE_LENGTH = 420;
const MAX_FEEDBACK_LENGTH = 2400;
export function createIterationMemory(): IterationMemory {
return {
updatedThroughIteration: 0,
bestScore: -1,
latestScore: 0,
latestMaxScore: 12,
latestVerdict: "",
activeFindings: [],
resolvedDimensions: [],
latestFeedback: "",
};
}
export function optimizeIterationMemory(
memory: IterationMemory,
judgeResult: JudgeFeedbackSummary,
iteration: number,
latestFeedback: string,
): IterationMemory {
const latestScore = finiteNumber(judgeResult.total_score, 0);
const latestMaxScore = finiteNumber(judgeResult.max_score, 12);
const previousFindings = new Map(
memory.activeFindings.map((finding) => [finding.dimension, finding]),
);
const activeFindings: IterationMemoryFinding[] = [];
const resolvedDimensions = new Set(memory.resolvedDimensions);
for (const [dimension, rawValue] of Object.entries(judgeResult.dimensions || {})) {
const score = Number(rawValue?.score);
if (!Number.isFinite(score)) continue;
if (score >= 2) {
if (previousFindings.delete(dimension)) {
resolvedDimensions.add(dimension);
}
continue;
}
const previous = previousFindings.get(dimension);
activeFindings.push({
dimension,
score,
message: compactText(rawValue?.explanation || judgeResult.rationale || judgeResult.summary || ""),
firstSeenIteration: previous?.firstSeenIteration ?? iteration,
lastSeenIteration: iteration,
hits: (previous?.hits ?? 0) + 1,
});
previousFindings.delete(dimension);
resolvedDimensions.delete(dimension);
}
for (const stale of previousFindings.values()) {
activeFindings.push(stale);
}
activeFindings.sort((a, b) => {
const scoreDelta = (a.score ?? 99) - (b.score ?? 99);
if (scoreDelta !== 0) return scoreDelta;
const hitDelta = b.hits - a.hits;
if (hitDelta !== 0) return hitDelta;
return b.lastSeenIteration - a.lastSeenIteration;
});
return {
updatedThroughIteration: iteration,
bestScore: Math.max(memory.bestScore, latestScore),
latestScore,
latestMaxScore,
latestVerdict: String(judgeResult.verdict || ""),
activeFindings: activeFindings.slice(0, MAX_FINDINGS),
resolvedDimensions: Array.from(resolvedDimensions).sort(),
latestFeedback: compactText(latestFeedback, MAX_FEEDBACK_LENGTH),
};
}
export function buildIterationMemoryPrompt(memory: IterationMemory): string {
if (!memory.updatedThroughIteration) return "";
const findings = memory.activeFindings.length
? memory.activeFindings.map((finding) => {
const score = finding.score == null ? "?" : String(finding.score);
const repeated = finding.hits > 1 ? `, repeated ${finding.hits}x` : "";
return [
`- ${finding.dimension} (score=${score}/2, first_seen=${finding.firstSeenIteration}, last_seen=${finding.lastSeenIteration}${repeated}):`,
` ${finding.message || "No explanation provided by Judge."}`,
].join("\n");
})
: ["- No active weak dimensions remain. Keep the previous fixes intact."];
const resolved = memory.resolvedDimensions.length
? "RESOLVED DIMENSIONS: " + memory.resolvedDimensions.join(", ")
: "";
return [
`=== ITERATION MEMORY (optimized through iteration ${memory.updatedThroughIteration}) ===`,
`LATEST SCORE: ${memory.latestScore}/${memory.latestMaxScore}`,
`BEST SCORE SO FAR: ${memory.bestScore}/${memory.latestMaxScore}`,
memory.latestVerdict ? `LATEST VERDICT: ${memory.latestVerdict}` : "",
"",
"ACTIVE ISSUES TO FIX BEFORE THE NEXT ITERATION:",
...findings,
"",
resolved,
"",
"MEMORY RULES:",
"- Fix active issues before changing unrelated fields.",
"- Do not reintroduce issues listed as resolved.",
"- Keep the output grounded in the real evidence; remove unsupported verbs instead of inventing actions.",
"",
memory.latestFeedback,
"=== END ITERATION MEMORY ===",
]
.filter(Boolean)
.join("\n");
}
function finiteNumber(value: unknown, fallback: number): number {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : fallback;
}
function compactText(value: unknown, limit = MAX_MESSAGE_LENGTH): string {
const text = String(value || "")
.replace(/\s+/g, " ")
.trim();
return text.length > limit ? text.slice(0, limit - 15).trimEnd() + "...(truncated)" : text;
}
|