Add interactive mode with verbose checking (#23)
* add back interactive mode, todo: print instructions for running w/ command and fix optional error message on command * add verbose checking
This commit is contained in:
parent
091a95d059
commit
73af777bc2
3 changed files with 44 additions and 8 deletions
21
src/index.ts
21
src/index.ts
|
|
@ -7,7 +7,8 @@ import {
|
||||||
printClientResults,
|
printClientResults,
|
||||||
runServerConformanceTest,
|
runServerConformanceTest,
|
||||||
printServerResults,
|
printServerResults,
|
||||||
printServerSummary
|
printServerSummary,
|
||||||
|
runInteractiveMode
|
||||||
} from './runner';
|
} from './runner';
|
||||||
import { listScenarios, listClientScenarios } from './scenarios';
|
import { listScenarios, listClientScenarios } from './scenarios';
|
||||||
import { ConformanceCheck } from './types';
|
import { ConformanceCheck } from './types';
|
||||||
|
|
@ -24,8 +25,10 @@ program
|
||||||
// Client command - tests a client implementation against scenarios
|
// Client command - tests a client implementation against scenarios
|
||||||
program
|
program
|
||||||
.command('client')
|
.command('client')
|
||||||
.description('Run conformance tests against a client implementation')
|
.description(
|
||||||
.requiredOption('--command <command>', 'Command to run the client')
|
'Run conformance tests against a client implementation or start interactive mode'
|
||||||
|
)
|
||||||
|
.option('--command <command>', 'Command to run the client')
|
||||||
.requiredOption('--scenario <scenario>', 'Scenario to test')
|
.requiredOption('--scenario <scenario>', 'Scenario to test')
|
||||||
.option('--timeout <ms>', 'Timeout in milliseconds', '30000')
|
.option('--timeout <ms>', 'Timeout in milliseconds', '30000')
|
||||||
.option('--verbose', 'Show verbose output')
|
.option('--verbose', 'Show verbose output')
|
||||||
|
|
@ -34,10 +37,20 @@ program
|
||||||
// Validate options with Zod
|
// Validate options with Zod
|
||||||
const validated = ClientOptionsSchema.parse(options);
|
const validated = ClientOptionsSchema.parse(options);
|
||||||
|
|
||||||
|
// If no command provided, run in interactive mode
|
||||||
|
if (!validated.command) {
|
||||||
|
await runInteractiveMode(
|
||||||
|
validated.scenario,
|
||||||
|
validated.verbose ?? false
|
||||||
|
);
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Otherwise run conformance test
|
||||||
const result = await runConformanceTest(
|
const result = await runConformanceTest(
|
||||||
validated.command,
|
validated.command,
|
||||||
validated.scenario,
|
validated.scenario,
|
||||||
validated.timeout
|
validated.timeout ?? 30000
|
||||||
);
|
);
|
||||||
|
|
||||||
const { failed } = printClientResults(
|
const { failed } = printClientResults(
|
||||||
|
|
|
||||||
|
|
@ -170,7 +170,10 @@ export function printClientResults(
|
||||||
return { passed, failed, denominator };
|
return { passed, failed, denominator };
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function runInteractiveMode(scenarioName: string): Promise<void> {
|
export async function runInteractiveMode(
|
||||||
|
scenarioName: string,
|
||||||
|
verbose: boolean = false
|
||||||
|
): Promise<void> {
|
||||||
await ensureResultsDir();
|
await ensureResultsDir();
|
||||||
const resultDir = createResultDir(scenarioName);
|
const resultDir = createResultDir(scenarioName);
|
||||||
await fs.mkdir(resultDir, { recursive: true });
|
await fs.mkdir(resultDir, { recursive: true });
|
||||||
|
|
@ -193,7 +196,11 @@ export async function runInteractiveMode(scenarioName: string): Promise<void> {
|
||||||
JSON.stringify(checks, null, 2)
|
JSON.stringify(checks, null, 2)
|
||||||
);
|
);
|
||||||
|
|
||||||
console.log(`\nChecks:\n${JSON.stringify(checks, null, 2)}`);
|
if (verbose) {
|
||||||
|
console.log(`\nChecks:\n${JSON.stringify(checks, null, 2)}`);
|
||||||
|
} else {
|
||||||
|
console.log(`\nChecks:\n${formatPrettyChecks(checks)}`);
|
||||||
|
}
|
||||||
console.log(`\nChecks saved to ${resultDir}/checks.json`);
|
console.log(`\nChecks saved to ${resultDir}/checks.json`);
|
||||||
|
|
||||||
await scenario.stop();
|
await scenario.stop();
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ import { getScenario, getClientScenario } from './scenarios';
|
||||||
|
|
||||||
// Client command options schema
|
// Client command options schema
|
||||||
export const ClientOptionsSchema = z.object({
|
export const ClientOptionsSchema = z.object({
|
||||||
command: z.string().min(1, 'Command cannot be empty'),
|
command: z.string().min(1, 'Command cannot be empty').optional(),
|
||||||
scenario: z
|
scenario: z
|
||||||
.string()
|
.string()
|
||||||
.min(1, 'Scenario cannot be empty')
|
.min(1, 'Scenario cannot be empty')
|
||||||
|
|
@ -21,7 +21,8 @@ export const ClientOptionsSchema = z.object({
|
||||||
.number()
|
.number()
|
||||||
.positive('Timeout must be a positive number')
|
.positive('Timeout must be a positive number')
|
||||||
.int('Timeout must be an integer')
|
.int('Timeout must be an integer')
|
||||||
),
|
)
|
||||||
|
.optional(),
|
||||||
verbose: z.boolean().optional()
|
verbose: z.boolean().optional()
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -42,3 +43,18 @@ export const ServerOptionsSchema = z.object({
|
||||||
});
|
});
|
||||||
|
|
||||||
export type ServerOptions = z.infer<typeof ServerOptionsSchema>;
|
export type ServerOptions = z.infer<typeof ServerOptionsSchema>;
|
||||||
|
|
||||||
|
// Interactive command options schema
|
||||||
|
export const InteractiveOptionsSchema = z.object({
|
||||||
|
scenario: z
|
||||||
|
.string()
|
||||||
|
.min(1, 'Scenario cannot be empty')
|
||||||
|
.refine(
|
||||||
|
(scenario) => getScenario(scenario) !== undefined,
|
||||||
|
(scenario) => ({
|
||||||
|
message: `Unknown scenario '${scenario}'`
|
||||||
|
})
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
export type InteractiveOptions = z.infer<typeof InteractiveOptionsSchema>;
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue