Add interactive mode with verbose checking (#23)

* add back interactive mode, todo: print instructions for running w/ command and fix optional error message on command

* add verbose checking
This commit is contained in:
Paul Carleton 2025-11-11 17:49:36 +00:00 • committed by GitHub
commit 73af777bc2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 44 additions and 8 deletions

View file

@ -7,7 +7,8 @@ import {
printClientResults,
runServerConformanceTest,
printServerResults,
printServerSummary
printServerSummary,
runInteractiveMode
} from './runner';
import { listScenarios, listClientScenarios } from './scenarios';
import { ConformanceCheck } from './types';
@ -24,8 +25,10 @@ program
// Client command - tests a client implementation against scenarios
program
.command('client')
.description('Run conformance tests against a client implementation')
.requiredOption('--command <command>', 'Command to run the client')
.description(
'Run conformance tests against a client implementation or start interactive mode'
)
.option('--command <command>', 'Command to run the client')
.requiredOption('--scenario <scenario>', 'Scenario to test')
.option('--timeout <ms>', 'Timeout in milliseconds', '30000')
.option('--verbose', 'Show verbose output')
@ -34,10 +37,20 @@ program
// Validate options with Zod
const validated = ClientOptionsSchema.parse(options);
// If no command provided, run in interactive mode
if (!validated.command) {
await runInteractiveMode(
validated.scenario,
validated.verbose ?? false
);
process.exit(0);
}
// Otherwise run conformance test
const result = await runConformanceTest(
validated.command,
validated.scenario,
validated.timeout
validated.timeout ?? 30000
);
const { failed } = printClientResults(

View file

@ -170,7 +170,10 @@ export function printClientResults(
return { passed, failed, denominator };
}
export async function runInteractiveMode(scenarioName: string): Promise<void> {
export async function runInteractiveMode(
scenarioName: string,
verbose: boolean = false
): Promise<void> {
await ensureResultsDir();
const resultDir = createResultDir(scenarioName);
await fs.mkdir(resultDir, { recursive: true });
@ -193,7 +196,11 @@ export async function runInteractiveMode(scenarioName: string): Promise<void> {
JSON.stringify(checks, null, 2)
);
console.log(`\nChecks:\n${JSON.stringify(checks, null, 2)}`);
if (verbose) {
console.log(`\nChecks:\n${JSON.stringify(checks, null, 2)}`);
} else {
console.log(`\nChecks:\n${formatPrettyChecks(checks)}`);
}
console.log(`\nChecks saved to ${resultDir}/checks.json`);
await scenario.stop();

View file

@ -3,7 +3,7 @@ import { getScenario, getClientScenario } from './scenarios';
// Client command options schema
export const ClientOptionsSchema = z.object({
command: z.string().min(1, 'Command cannot be empty'),
command: z.string().min(1, 'Command cannot be empty').optional(),
scenario: z
.string()
.min(1, 'Scenario cannot be empty')
@ -21,7 +21,8 @@ export const ClientOptionsSchema = z.object({
.number()
.positive('Timeout must be a positive number')
.int('Timeout must be an integer')
),
)
.optional(),
verbose: z.boolean().optional()
});
@ -42,3 +43,18 @@ export const ServerOptionsSchema = z.object({
});
export type ServerOptions = z.infer<typeof ServerOptionsSchema>;
// Interactive command options schema
export const InteractiveOptionsSchema = z.object({
scenario: z
.string()
.min(1, 'Scenario cannot be empty')
.refine(
(scenario) => getScenario(scenario) !== undefined,
(scenario) => ({
message: `Unknown scenario '${scenario}'`
})
)
});
export type InteractiveOptions = z.infer<typeof InteractiveOptionsSchema>;