import { TierScorecard, CheckStatus, ConformanceResult } from './types'; import { DATED_SPEC_VERSIONS, DRAFT_PROTOCOL_VERSION } from '../types'; const COLORS = { RESET: '\x1b[0m', GREEN: '\x1b[32m', YELLOW: '\x1b[33m', RED: '\x1b[31m', BLUE: '\x1b[36m', BOLD: '\x1b[1m', DIM: '\x1b[2m' }; function statusIcon(status: CheckStatus): string { switch (status) { case 'pass': return `${COLORS.GREEN}\u2713${COLORS.RESET}`; case 'fail': return `${COLORS.RED}\u2717${COLORS.RESET}`; case 'partial': return `${COLORS.YELLOW}~${COLORS.RESET}`; case 'skipped': return `${COLORS.DIM}-${COLORS.RESET}`; } } const TIER_SPEC_VERSIONS = DATED_SPEC_VERSIONS; const INFO_SPEC_VERSIONS = [DRAFT_PROTOCOL_VERSION, 'extension'] as const; type Cell = { passed: number; total: number }; interface MatrixRow { cells: Map; /** Unique scenario counts for tier-scoring versions only. */ tierUnique: Cell; /** Unique scenario counts for informational versions only. */ infoUnique: Cell; } const INFO_SET = new Set(INFO_SPEC_VERSIONS); function newRow(): MatrixRow { return { cells: new Map(), tierUnique: { passed: 0, total: 0 }, infoUnique: { passed: 0, total: 0 } }; } interface ConformanceMatrix { server: MatrixRow; clientCore: MatrixRow; clientAuth: MatrixRow; } function buildConformanceMatrix( server: ConformanceResult, client: ConformanceResult ): ConformanceMatrix { const matrix: ConformanceMatrix = { server: newRow(), clientCore: newRow(), clientAuth: newRow() }; function addToRow( row: MatrixRow, d: { passed: boolean; specVersions?: string[] } ) { const versions = d.specVersions ?? ['unknown']; const isTierScoring = versions.some((v) => !INFO_SET.has(v)); const bucket = isTierScoring ? row.tierUnique : row.infoUnique; bucket.total++; if (d.passed) bucket.passed++; for (const v of versions) { const cell = row.cells.get(v) ?? { passed: 0, total: 0 }; cell.total++; if (d.passed) cell.passed++; row.cells.set(v, cell); } } for (const d of server.details) { addToRow(matrix.server, d); } for (const d of client.details) { const row = d.scenario.startsWith('auth/') ? matrix.clientAuth : matrix.clientCore; addToRow(row, d); } return matrix; } function formatCell(cell: Cell | undefined): string { if (!cell || cell.total === 0) return '\u2014'; return `${cell.passed}/${cell.total}`; } function formatRate(cell: Cell): string { if (cell.total === 0) return '0/0'; return `${cell.passed}/${cell.total} (${Math.round((cell.passed / cell.total) * 100)}%)`; } /** * Under a requirement set the spec-version matrix is the wrong picture: the set * already names exactly what is scored, so splitting it by applicability tag and * filing part of it as "informational" would contradict the score. */ /** * A run at one revision's wire and a run at another's are distinct * measurements, so entries are never deduplicated across revisions — the * counts here must match the JSON, and a scenario that fails on one wire but * passes on the other must say which. The label carries the revision whenever * one is recorded. */ function withRevision(d: { scenario: string; revision?: string }): string { return d.revision ? `${d.scenario} [${d.revision}]` : d.scenario; } function requirementsSummary(scorecard: TierScorecard): string[] { const revisions = scorecard.requirements_revisions ?? []; const rows: [string, ConformanceResult][] = [ ['Server', scorecard.checks.conformance as ConformanceResult], ['Client', scorecard.checks.client_conformance as ConformanceResult] ]; const lines = [ '', `Scored against the ${revisions.join(' and ')} requirement set${revisions.length > 1 ? 's' : ''}, each run at its own wire version.`, '', '| | Required | Passed | |', '|---|---|---|---|' ]; for (const [label, result] of rows) { if (result.error) { lines.push(`| ${label} | — | — | not measured |`); continue; } if (result.status === 'skipped') { lines.push(`| ${label} | — | — | skipped |`); continue; } lines.push( `| ${label} | ${result.total} | ${result.passed} | ${Math.round(result.pass_rate * 100)}% |` ); } // A requirement set is small and every entry is named, so a bare count is not // actionable: say which ones failed. const failed = rows.flatMap(([label, result]) => result.details .filter((d) => !d.passed && !d.notScoredReason) .map((d) => `${label.toLowerCase()}: ${withRevision(d)}`) ); if (failed.length > 0) { lines.push(''); lines.push(`_Failing (${failed.length}):_`); lines.push(''); failed.forEach((f) => lines.push(`- ${f}`)); } // Run and reported, but deliberately outside the score. const notScored = rows.flatMap(([label, result]) => result.details .filter((d) => d.notScoredReason) .map((d) => ({ ...d, leg: label.toLowerCase() })) ); if (notScored.length > 0) { const failing = notScored.filter((d) => !d.passed); lines.push(''); lines.push( `_Not scored (${notScored.length} run, ${failing.length} failing). These do not affect the tier:_` ); lines.push(''); lines.push('| Scenario | Leg | Result | Why not scored |'); lines.push('|---|---|---|---|'); for (const d of notScored) { lines.push( `| ${withRevision(d)} | ${d.leg} | ${d.passed ? 'pass' : 'fail'} | ${d.notScoredReason} |` ); } } return lines; } export function formatJson(scorecard: TierScorecard): string { return JSON.stringify(scorecard, null, 2); } export function formatMarkdown(scorecard: TierScorecard): string { const lines: string[] = []; const c = scorecard.checks; lines.push(`# Tier Assessment: Tier ${scorecard.implied_tier.tier}`); lines.push(''); lines.push(`**Repo**: ${scorecard.repo}`); if (scorecard.branch) lines.push(`**Branch**: ${scorecard.branch}`); if (scorecard.version) lines.push(`**Version**: ${scorecard.version}`); if (scorecard.requirements_revisions) lines.push( `**Requirements**: ${scorecard.requirements_revisions.join(', ')} (frozen; each run at its own wire)` ); lines.push(`**Timestamp**: ${scorecard.timestamp}`); lines.push(''); lines.push('## Check Results'); lines.push(''); // Conformance matrix const matrix = buildConformanceMatrix( c.conformance as ConformanceResult, c.client_conformance as ConformanceResult ); if (scorecard.requirements_revisions) { lines.push(...requirementsSummary(scorecard)); } else { // Tier-scoring matrix lines.push(''); lines.push(`| | ${TIER_SPEC_VERSIONS.join(' | ')} | All* |`); lines.push(`|---|${TIER_SPEC_VERSIONS.map(() => '---|').join('')}---|`); const mdRows: [string, MatrixRow][] = [ ['Server', matrix.server], ['Client: Core', matrix.clientCore], ['Client: Auth', matrix.clientAuth] ]; for (const [label, row] of mdRows) { lines.push( `| ${label} | ${TIER_SPEC_VERSIONS.map((v) => formatCell(row.cells.get(v))).join(' | ')} | ${formatRate(row.tierUnique)} |` ); } lines.push(''); lines.push( '_* unique scenarios — a scenario may apply to multiple spec versions_' ); // Informational matrix (draft/extension) const hasInfoMd = mdRows.some(([, row]) => INFO_SPEC_VERSIONS.some((v) => { const cell = row.cells.get(v); return cell && cell.total > 0; }) ); if (hasInfoMd) { lines.push(''); lines.push('_Informational (not scored for tier):_'); lines.push(''); lines.push(`| | ${INFO_SPEC_VERSIONS.join(' | ')} |`); lines.push(`|---|${INFO_SPEC_VERSIONS.map(() => '---|').join('')}`); for (const [label, row] of mdRows) { const hasData = INFO_SPEC_VERSIONS.some((v) => { const cell = row.cells.get(v); return cell && cell.total > 0; }); if (!hasData) continue; lines.push( `| ${label} | ${INFO_SPEC_VERSIONS.map((v) => formatCell(row.cells.get(v))).join(' | ')} |` ); } } } lines.push(''); lines.push('| Check | Status | Detail |'); lines.push('|-------|--------|--------|'); lines.push( `| Labels | ${c.labels.status} | ${c.labels.present}/${c.labels.required} required labels${c.labels.missing.length > 0 ? ` (missing: ${c.labels.missing.join(', ')})` : ''} |` ); lines.push( `| Triage | ${c.triage.status} | ${Math.round(c.triage.compliance_rate * 100)}% within 2BD, median ${c.triage.median_hours}h, p95 ${c.triage.p95_hours}h |` ); lines.push( `| P0 Resolution | ${c.p0_resolution.status} | ${c.p0_resolution.open_p0s} open, ${c.p0_resolution.closed_within_7d}/${c.p0_resolution.closed_total} closed within 7d |` ); lines.push( `| Stable Release | ${c.stable_release.status} | ${c.stable_release.version || 'none'} (stable: ${c.stable_release.is_stable}) |` ); lines.push( `| Policy Signals | ${c.policy_signals.status} | ${Object.entries( c.policy_signals.files ) .map(([f, e]) => `${f}: ${e ? '\u2713' : '\u2717'}`) .join(', ')} |` ); lines.push( `| Spec Tracking | ${c.spec_tracking.status} | ${c.spec_tracking.days_gap !== null ? `${c.spec_tracking.days_gap}d gap` : 'N/A'} |` ); lines.push(''); if (scorecard.implied_tier.tier1_blockers.length > 0) { lines.push('## Tier 1 Blockers'); lines.push(''); for (const blocker of scorecard.implied_tier.tier1_blockers) { lines.push(`- ${blocker}`); } lines.push(''); } lines.push(`> ${scorecard.implied_tier.note}`); return lines.join('\n'); } export function formatTerminal(scorecard: TierScorecard): void { const c = scorecard.checks; const tier = scorecard.implied_tier.tier; const tierColor = tier === 1 ? COLORS.GREEN : tier === 2 ? COLORS.YELLOW : COLORS.RED; console.log( `\n${COLORS.BOLD}Tier Assessment: ${tierColor}Tier ${tier}${COLORS.RESET}\n` ); console.log(`Repo: ${scorecard.repo}`); if (scorecard.branch) console.log(`Branch: ${scorecard.branch}`); if (scorecard.version) console.log(`Version: ${scorecard.version}`); if (scorecard.requirements_revisions) console.log( `Requires: ${scorecard.requirements_revisions.join(', ')} (frozen, each at its own wire)` ); console.log(`Timestamp: ${scorecard.timestamp}\n`); console.log(`${COLORS.BOLD}Conformance:${COLORS.RESET}\n`); if (scorecard.requirements_revisions) { console.log( ` ${COLORS.DIM}Scored against ${scorecard.requirements_revisions.join(' and ')}, each run at its own wire version.${COLORS.RESET}\n` ); const reqRows: [string, ConformanceResult][] = [ ['Server', c.conformance as ConformanceResult], ['Client', c.client_conformance as ConformanceResult] ]; for (const [label, result] of reqRows) { if (result.status === 'skipped') { console.log(` ${label.padEnd(8)} skipped`); continue; } if (result.error) { console.log( ` ${label.padEnd(8)} ${COLORS.RED}not measured${COLORS.RESET}: ${result.error}` ); continue; } console.log( ` ${label.padEnd(8)} ${result.passed}/${result.total} required scenarios (${Math.round(result.pass_rate * 100)}%)` ); } const failedReq = reqRows.flatMap(([label, result]) => result.details .filter((d) => !d.passed && !d.notScoredReason) .map((d) => `${label.toLowerCase()}: ${withRevision(d)}`) ); if (failedReq.length > 0) { console.log( `\n ${COLORS.RED}Failing (${failedReq.length}):${COLORS.RESET}` ); failedReq.forEach((f) => console.log(` ${COLORS.RED}\u2717${COLORS.RESET} ${f}`) ); } const notScoredReq = reqRows.flatMap(([, result]) => result.details.filter((d) => d.notScoredReason) ); if (notScoredReq.length > 0) { const failing = notScoredReq.filter((d) => !d.passed); console.log( `\n ${COLORS.DIM}Not scored (${notScoredReq.length} run, ${failing.length} failing, no effect on tier):${COLORS.RESET}` ); for (const d of notScoredReq) { const mark = d.passed ? `${COLORS.GREEN}\u2713${COLORS.RESET}` : `${COLORS.DIM}\u2717${COLORS.RESET}`; console.log( ` ${mark} ${withRevision(d)} ${COLORS.DIM}(${d.notScoredReason})${COLORS.RESET}` ); } } console.log(''); } else { // Conformance matrix const matrix = buildConformanceMatrix( c.conformance as ConformanceResult, c.client_conformance as ConformanceResult ); const vw = 10; // column width for version cells const lw = 14; // label column width const tw = 16; // total column width const rp = (s: string, w: number) => s.padStart(w); const lp = (s: string, w: number) => s.padEnd(w); // Tier-scoring matrix (date-versioned specs only) console.log( ` ${COLORS.DIM}${lp('', lw + 2)} ${TIER_SPEC_VERSIONS.map((v) => rp(v, vw)).join(' ')} ${rp('All*', tw)}${COLORS.RESET}` ); const rows: [string, MatrixRow, CheckStatus | null, boolean][] = [ ['Server', matrix.server, c.conformance.status, true], ['Client: Core', matrix.clientCore, null, false], ['Client: Auth', matrix.clientAuth, null, false] ]; for (const [label, row, status, bold] of rows) { const icon = status ? statusIcon(status) + ' ' : ' '; const b = bold ? COLORS.BOLD : ''; const r = bold ? COLORS.RESET : ''; console.log( ` ${icon}${b}${lp(label, lw)}${r} ${TIER_SPEC_VERSIONS.map((v) => rp(formatCell(row.cells.get(v)), vw)).join(' ')} ${b}${rp(formatRate(row.tierUnique), tw)}${r}` ); } // Client total line (tier-scoring only) const clientTierTotal: Cell = { passed: matrix.clientCore.tierUnique.passed + matrix.clientAuth.tierUnique.passed, total: matrix.clientCore.tierUnique.total + matrix.clientAuth.tierUnique.total }; console.log( ` ${statusIcon(c.client_conformance.status)} ${COLORS.BOLD}${lp('Client Total', lw)}${COLORS.RESET} ${' '.repeat(TIER_SPEC_VERSIONS.length * (vw + 1) - 1)} ${COLORS.BOLD}${rp(formatRate(clientTierTotal), tw)}${COLORS.RESET}` ); console.log( `\n ${COLORS.DIM}* unique scenarios — a scenario may apply to multiple spec versions${COLORS.RESET}` ); // Informational matrix (draft/extension) — only if there are any const hasInfo = rows.some(([, row]) => INFO_SPEC_VERSIONS.some((v) => { const cell = row.cells.get(v); return cell && cell.total > 0; }) ); if (hasInfo) { console.log(`\n Informational (not scored for tier):\n`); console.log( ` ${COLORS.DIM}${lp('', lw + 2)} ${INFO_SPEC_VERSIONS.map((v) => rp(v, vw)).join(' ')}${COLORS.RESET}` ); for (const [label, row, , bold] of rows) { const hasData = INFO_SPEC_VERSIONS.some((v) => { const cell = row.cells.get(v); return cell && cell.total > 0; }); if (!hasData) continue; const b = bold ? COLORS.BOLD : ''; const r = bold ? COLORS.RESET : ''; console.log( ` ${b}${lp(label, lw)}${r} ${INFO_SPEC_VERSIONS.map((v) => rp(formatCell(row.cells.get(v)), vw)).join(' ')}` ); } } } console.log(`\n${COLORS.BOLD}Repository Health:${COLORS.RESET}\n`); console.log( ` ${statusIcon(c.labels.status)} Labels ${c.labels.present}/${c.labels.required} required labels` ); if (c.labels.missing.length > 0) console.log( ` ${COLORS.DIM}Missing: ${c.labels.missing.join(', ')}${COLORS.RESET}` ); console.log( ` ${statusIcon(c.triage.status)} Triage ${Math.round(c.triage.compliance_rate * 100)}% within 2BD (${c.triage.total_issues} issues, median ${c.triage.median_hours}h)` ); console.log( ` ${statusIcon(c.p0_resolution.status)} P0 Resolution ${c.p0_resolution.open_p0s} open, ${c.p0_resolution.closed_within_7d}/${c.p0_resolution.closed_total} closed within 7d` ); if (c.p0_resolution.open_p0_details.length > 0) { for (const p0 of c.p0_resolution.open_p0_details) { console.log( ` ${COLORS.RED}#${p0.number} (${p0.age_days}d old): ${p0.title}${COLORS.RESET}` ); } } console.log( ` ${statusIcon(c.stable_release.status)} Stable Release ${c.stable_release.version || 'none'}` ); console.log( ` ${statusIcon(c.policy_signals.status)} Policy Signals ${Object.entries( c.policy_signals.files ) .map(([f, e]) => `${e ? '\u2713' : '\u2717'} ${f}`) .join(', ')}` ); console.log( ` ${statusIcon(c.spec_tracking.status)} Spec Tracking ${c.spec_tracking.days_gap !== null ? `${c.spec_tracking.days_gap}d gap` : 'N/A'}` ); if (scorecard.implied_tier.tier1_blockers.length > 0) { console.log(`\n${COLORS.BOLD}Tier 1 Blockers:${COLORS.RESET}`); for (const blocker of scorecard.implied_tier.tier1_blockers) { console.log(` ${COLORS.RED}\u2022${COLORS.RESET} ${blocker}`); } } console.log(`\n${COLORS.DIM}${scorecard.implied_tier.note}${COLORS.RESET}\n`); }