Add interactive mode with verbose checking (#23)

* add back interactive mode, todo: print instructions for running w/ command and fix optional error message on command

* add verbose checking
This commit is contained in:
Paul Carleton 2025-11-11 17:49:36 +00:00 • committed by GitHub
commit 73af777bc2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 44 additions and 8 deletions

View file

@ -7,7 +7,8 @@ import {
printClientResults, printClientResults,
runServerConformanceTest, runServerConformanceTest,
printServerResults, printServerResults,
printServerSummary printServerSummary,
runInteractiveMode
} from './runner'; } from './runner';
import { listScenarios, listClientScenarios } from './scenarios'; import { listScenarios, listClientScenarios } from './scenarios';
import { ConformanceCheck } from './types'; import { ConformanceCheck } from './types';
@ -24,8 +25,10 @@ program
// Client command - tests a client implementation against scenarios // Client command - tests a client implementation against scenarios
program program
.command('client') .command('client')
.description('Run conformance tests against a client implementation') .description(
.requiredOption('--command <command>', 'Command to run the client') 'Run conformance tests against a client implementation or start interactive mode'
)
.option('--command <command>', 'Command to run the client')
.requiredOption('--scenario <scenario>', 'Scenario to test') .requiredOption('--scenario <scenario>', 'Scenario to test')
.option('--timeout <ms>', 'Timeout in milliseconds', '30000') .option('--timeout <ms>', 'Timeout in milliseconds', '30000')
.option('--verbose', 'Show verbose output') .option('--verbose', 'Show verbose output')
@ -34,10 +37,20 @@ program
// Validate options with Zod // Validate options with Zod
const validated = ClientOptionsSchema.parse(options); const validated = ClientOptionsSchema.parse(options);
// If no command provided, run in interactive mode
if (!validated.command) {
await runInteractiveMode(
validated.scenario,
validated.verbose ?? false
);
process.exit(0);
}
// Otherwise run conformance test
const result = await runConformanceTest( const result = await runConformanceTest(
validated.command, validated.command,
validated.scenario, validated.scenario,
validated.timeout validated.timeout ?? 30000
); );
const { failed } = printClientResults( const { failed } = printClientResults(

View file

@ -170,7 +170,10 @@ export function printClientResults(
return { passed, failed, denominator }; return { passed, failed, denominator };
} }
export async function runInteractiveMode(scenarioName: string): Promise<void> { export async function runInteractiveMode(
scenarioName: string,
verbose: boolean = false
): Promise<void> {
await ensureResultsDir(); await ensureResultsDir();
const resultDir = createResultDir(scenarioName); const resultDir = createResultDir(scenarioName);
await fs.mkdir(resultDir, { recursive: true }); await fs.mkdir(resultDir, { recursive: true });
@ -193,7 +196,11 @@ export async function runInteractiveMode(scenarioName: string): Promise<void> {
JSON.stringify(checks, null, 2) JSON.stringify(checks, null, 2)
); );
console.log(`\nChecks:\n${JSON.stringify(checks, null, 2)}`); if (verbose) {
console.log(`\nChecks:\n${JSON.stringify(checks, null, 2)}`);
} else {
console.log(`\nChecks:\n${formatPrettyChecks(checks)}`);
}
console.log(`\nChecks saved to ${resultDir}/checks.json`); console.log(`\nChecks saved to ${resultDir}/checks.json`);
await scenario.stop(); await scenario.stop();

View file

@ -3,7 +3,7 @@ import { getScenario, getClientScenario } from './scenarios';
// Client command options schema // Client command options schema
export const ClientOptionsSchema = z.object({ export const ClientOptionsSchema = z.object({
command: z.string().min(1, 'Command cannot be empty'), command: z.string().min(1, 'Command cannot be empty').optional(),
scenario: z scenario: z
.string() .string()
.min(1, 'Scenario cannot be empty') .min(1, 'Scenario cannot be empty')
@ -21,7 +21,8 @@ export const ClientOptionsSchema = z.object({
.number() .number()
.positive('Timeout must be a positive number') .positive('Timeout must be a positive number')
.int('Timeout must be an integer') .int('Timeout must be an integer')
), )
.optional(),
verbose: z.boolean().optional() verbose: z.boolean().optional()
}); });
@ -42,3 +43,18 @@ export const ServerOptionsSchema = z.object({
}); });
export type ServerOptions = z.infer<typeof ServerOptionsSchema>; export type ServerOptions = z.infer<typeof ServerOptionsSchema>;
// Interactive command options schema
export const InteractiveOptionsSchema = z.object({
scenario: z
.string()
.min(1, 'Scenario cannot be empty')
.refine(
(scenario) => getScenario(scenario) !== undefined,
(scenario) => ({
message: `Unknown scenario '${scenario}'`
})
)
});
export type InteractiveOptions = z.infer<typeof InteractiveOptionsSchema>;