modelcontextprotocol/conformance
Watch
1
0
Fork
You've already forked conformance
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: recurse into subdirectories when parsing conformance output (#148)

Browse source 
parseOutputDir only read one level deep, so scenarios with '/' in their
name (e.g. auth/metadata-default) were stored in nested subdirectories
and never found. This caused tier-check to report 4/23 client conformance
when the actual pass rate was 23/23.

Also removes accidentally committed pnpm-lock.yaml.
This commit is contained in:
Felix Weinberger 2026-02-13 16:13:48 +00:00 • committed by GitHub
commit c82fd65cfa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 9 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/tier-check/checks/test-conformance-results.ts
View file

@ -1,6 +1,6 @@
import { execSync } from 'child_process';
import { mkdtempSync, readFileSync, readdirSync, existsSync } from 'fs';
import { join } from 'path';
import { mkdtempSync, readFileSync, existsSync, globSync } from 'fs';
import { join, dirname } from 'path';
import { tmpdir } from 'os';
import { ConformanceResult } from '../types';
import { listScenarios, listActiveClientScenarios } from '../../scenarios';
@ -26,10 +26,13 @@ function parseOutputDir(outputDir: string): ConformanceResult {
let totalPassed = 0;
let totalFailed = 0;
const entries = readdirSync(outputDir);
for (const scenarioName of entries) {
const checksPath = join(outputDir, scenarioName, 'checks.json');
if (!existsSync(checksPath)) continue;
// Find all checks.json files recursively to handle scenarios with '/' in
// their name (e.g. auth/metadata-default) which create nested subdirectories.
const checksFiles = globSync('**/checks.json', { cwd: outputDir });
for (const checksFile of checksFiles) {
const scenarioName = dirname(checksFile);
const checksPath = join(outputDir, checksFile);
try {
const checks: ConformanceCheck[] = JSON.parse(