use the diff package, aider inspired search

This commit is contained in:
Marc Goodner 2024-12-04 20:13:58 -08:00
commit b6e052946a
2 changed files with 210 additions and 113 deletions

View file

@ -12,6 +12,7 @@ import path from "path";
import os from 'os'; import os from 'os';
import { z } from "zod"; import { z } from "zod";
import { zodToJsonSchema } from "zod-to-json-schema"; import { zodToJsonSchema } from "zod-to-json-schema";
import { diffLines, createTwoFilesPatch } from 'diff';
// Command line argument parsing // Command line argument parsing
const args = process.argv.slice(2); const args = process.argv.slice(2);
@ -216,146 +217,238 @@ async function searchFiles(
return results; return results;
} }
interface DiffLine { // file editing and diffing utilities
type: 'context' | 'addition' | 'deletion'; function createUnifiedDiff(originalContent: string, newContent: string, filepath: string = 'file'): string {
content: string; return createTwoFilesPatch(
lineNumber: number; filepath,
filepath,
originalContent,
newContent,
'original',
'modified'
);
} }
function createUnifiedDiff(originalLines: string[], newLines: string[], contextSize: number = 3): string { // Utility functions for text normalization and matching
const differ = new Array<DiffLine>(); function normalizeLineEndings(text: string): string {
let lineNumber = 1; return text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
}
// Helper to add context lines
function addContext(lines: string[], start: number, count: number) { function normalizeWhitespace(text: string, preserveIndentation: boolean = true): string {
for (let i = 0; i < count && start + i < lines.length; i++) { if (!preserveIndentation) {
differ.push({ // Collapse all whitespace to single spaces if not preserving indentation
type: 'context', return text.replace(/\s+/g, ' ');
content: lines[start + i], }
lineNumber: start + i + 1
}); // Preserve line structure but normalize inline whitespace
} return text.split('\n').map(line => {
} // Preserve leading whitespace
const indent = line.match(/^[\s\t]*/)?.[0] || '';
// Find the differences using longest common subsequence // Normalize rest of line
const changes: Array<{type: 'context' | 'addition' | 'deletion', line: string, index: number}> = []; const content = line.slice(indent.length).trim().replace(/\s+/g, ' ');
let i = 0, j = 0; return indent + content;
}).join('\n');
while (i < originalLines.length || j < newLines.length) { }
if (i < originalLines.length && j < newLines.length && originalLines[i] === newLines[j]) {
changes.push({type: 'context', line: originalLines[i], index: i}); interface EditOptions {
i++; preserveIndentation?: boolean;
j++; normalizeWhitespace?: boolean;
} else { partialMatch?: boolean;
if (i < originalLines.length) { }
changes.push({type: 'deletion', line: originalLines[i], index: i});
i++; interface EditMatch {
} start: number;
if (j < newLines.length) { end: number;
changes.push({type: 'addition', line: newLines[j], index: j}); confidence: number;
j++; }
}
} function findBestMatch(content: string, searchText: string, options: EditOptions): EditMatch | null {
} const normalizedContent = normalizeLineEndings(content);
const normalizedSearch = normalizeLineEndings(searchText);
// Group changes into hunks with context
let currentHunk: DiffLine[] = []; // Try exact match first
let hunks: DiffLine[][] = []; const exactPos = normalizedContent.indexOf(normalizedSearch);
let lastChangeIndex = -1; if (exactPos !== -1) {
return {
for (let i = 0; i < changes.length; i++) { start: exactPos,
const change = changes[i]; end: exactPos + searchText.length,
confidence: 1.0
if (change.type !== 'context' || };
(lastChangeIndex >= 0 && i - lastChangeIndex <= contextSize * 2)) { }
if (change.type !== 'context') {
lastChangeIndex = i; // If whitespace normalization is enabled, try that next
} if (options.normalizeWhitespace) {
currentHunk.push({ const normContent = normalizeWhitespace(normalizedContent, options.preserveIndentation);
type: change.type, const normSearch = normalizeWhitespace(normalizedSearch, options.preserveIndentation);
content: change.line, const normPos = normContent.indexOf(normSearch);
lineNumber: change.index + 1
}); if (normPos !== -1) {
} else { // Find the corresponding position in original text
if (currentHunk.length > 0) { const beforeMatch = normContent.slice(0, normPos);
hunks.push(currentHunk); const originalPos = findOriginalPosition(content, beforeMatch);
currentHunk = []; return {
} start: originalPos,
} end: originalPos + searchText.length,
} confidence: 0.9
};
if (currentHunk.length > 0) { }
hunks.push(currentHunk); }
}
// If partial matching is enabled, try to find the best partial match
// Format the diff output if (options.partialMatch) {
let diffOutput = ''; const lines = normalizedContent.split('\n');
const searchLines = normalizedSearch.split('\n');
for (const hunk of hunks) {
const startLine = hunk[0].lineNumber; let bestMatch: EditMatch | null = null;
const endLine = hunk[hunk.length - 1].lineNumber; let bestScore = 0;
diffOutput += `@@ -${startLine},${endLine} @@\n`; // Sliding window search through the content
for (let i = 0; i < lines.length - searchLines.length + 1; i++) {
for (const line of hunk) { let matchScore = 0;
const prefix = line.type === 'addition' ? '+' : let matchLength = 0;
line.type === 'deletion' ? '-' : ' ';
diffOutput += `${prefix}${line.content}\n`; for (let j = 0; j < searchLines.length; j++) {
} const contentLine = options.normalizeWhitespace
? normalizeWhitespace(lines[i + j], options.preserveIndentation)
diffOutput += '\n'; : lines[i + j];
} const searchLine = options.normalizeWhitespace
? normalizeWhitespace(searchLines[j], options.preserveIndentation)
return diffOutput; : searchLines[j];
const similarity = calculateSimilarity(contentLine, searchLine);
matchScore += similarity;
matchLength += lines[i + j].length + 1; // +1 for newline
}
const averageScore = matchScore / searchLines.length;
if (averageScore > bestScore && averageScore > 0.7) { // Threshold for minimum match quality
bestScore = averageScore;
const start = lines.slice(0, i).reduce((acc, line) => acc + line.length + 1, 0);
bestMatch = {
start,
end: start + matchLength,
confidence: averageScore
};
}
}
return bestMatch;
}
return null;
}
function calculateSimilarity(str1: string, str2: string): number {
const len1 = str1.length;
const len2 = str2.length;
const matrix: number[][] = Array(len1 + 1).fill(null).map(() => Array(len2 + 1).fill(0));
for (let i = 0; i <= len1; i++) matrix[i][0] = i;
for (let j = 0; j <= len2; j++) matrix[0][j] = j;
for (let i = 1; i <= len1; i++) {
for (let j = 1; j <= len2; j++) {
const cost = str1[i - 1] === str2[j - 1] ? 0 : 1;
matrix[i][j] = Math.min(
matrix[i - 1][j] + 1,
matrix[i][j - 1] + 1,
matrix[i - 1][j - 1] + cost
);
}
}
const maxLength = Math.max(len1, len2);
return maxLength === 0 ? 1 : (maxLength - matrix[len1][len2]) / maxLength;
}
function findOriginalPosition(original: string, normalizedPrefix: string): number {
let origPos = 0;
let normPos = 0;
while (normPos < normalizedPrefix.length && origPos < original.length) {
if (normalizeWhitespace(original[origPos], true) === normalizedPrefix[normPos]) {
normPos++;
}
origPos++;
}
return origPos;
} }
// File editing utilities
async function applyFileEdits( async function applyFileEdits(
filePath: string, filePath: string,
edits: Array<{oldText: string, newText: string}>, edits: Array<{oldText: string, newText: string}>,
dryRun = false dryRun = false,
): Promise<string | string> { options: EditOptions = {
let content = await fs.readFile(filePath, 'utf-8'); preserveIndentation: true,
const originalLines = content.split('\n'); normalizeWhitespace: true,
partialMatch: true
}
): Promise<string> {
const content = await fs.readFile(filePath, 'utf-8');
let modifiedContent = content; let modifiedContent = content;
const failedEdits: Array<{edit: typeof edits[0], error: string}> = [];
const successfulEdits: Array<{edit: typeof edits[0], match: EditMatch}> = [];
// First, validate all edits can be applied // Sort edits by position (if found) to apply them in order
const positions = edits.map(edit => { for (const edit of edits) {
const pos = modifiedContent.indexOf(edit.oldText); const match = findBestMatch(modifiedContent, edit.oldText, options);
if (pos === -1) {
throw new Error(`Text not found:\n${edit.oldText}`); if (!match) {
failedEdits.push({
edit,
error: 'No suitable match found'
});
continue;
} }
return {
edit,
position: pos,
length: edit.oldText.length
};
});
// Sort positions in reverse order to apply from end to start // For low confidence matches in non-dry-run mode, we might want to throw
positions.sort((a, b) => b.position - a.position); if (!dryRun && match.confidence < 0.8) {
failedEdits.push({
edit,
error: `Match confidence too low: ${(match.confidence * 100).toFixed(1)}%`
});
continue;
}
successfulEdits.push({ edit, match });
}
// Sort successful edits by position (reverse order to maintain positions)
successfulEdits.sort((a, b) => b.match.start - a.match.start);
// Apply successful edits
for (const { edit, match } of successfulEdits) {
modifiedContent =
modifiedContent.slice(0, match.start) +
edit.newText +
modifiedContent.slice(match.end);
}
if (dryRun) { if (dryRun) {
// For dry run, create a unified diff preview let report = createUnifiedDiff(content, modifiedContent, filePath);
for (const {edit, position} of positions) {
modifiedContent = if (failedEdits.length > 0) {
modifiedContent.slice(0, position) + report += '\nFailed edits:\n' + failedEdits.map(({ edit, error }) =>
edit.newText + `- Error: ${error}\n Old text: ${edit.oldText.split('\n')[0]}...\n`
modifiedContent.slice(position + edit.oldText.length); ).join('\n');
} }
const modifiedLines = modifiedContent.split('\n'); if (successfulEdits.length > 0) {
return createUnifiedDiff(originalLines, modifiedLines); report += '\nSuccessful edits:\n' + successfulEdits.map(({ edit, match }) =>
} else { `- Match confidence: ${(match.confidence * 100).toFixed(1)}%\n Position: ${match.start}-${match.end}\n`
// Apply the edits ).join('\n');
for (const {edit, position} of positions) {
modifiedContent =
modifiedContent.slice(0, position) +
edit.newText +
modifiedContent.slice(position + edit.oldText.length);
} }
return modifiedContent;
return report;
} }
if (failedEdits.length > 0) {
const errors = failedEdits.map(({ error }) => error).join('\n');
throw new Error(`Some edits failed:\n${errors}`);
}
return modifiedContent;
} }
// Tool handlers // Tool handlers
@ -392,8 +485,10 @@ server.setRequestHandler(ListToolsRequestSchema, async () => {
{ {
name: "edit_file", name: "edit_file",
description: description:
"Make selective edits to a text file using search and replace with unified diff previews. " + "Make selective edits to a text file using line-based pattern matching and replacement. " +
"Handles both single-line and multi-line edits, with smart positioning to handle multiple edits simultaneously. " +
"Shows changes in standard unified diff format with context lines, similar to git diff. " + "Shows changes in standard unified diff format with context lines, similar to git diff. " +
"Provides detailed diff output for failed matches to aid debugging. " +
"Use dry run mode to preview changes in patch format before applying them. " + "Use dry run mode to preview changes in patch format before applying them. " +
"Only works within allowed directories.", "Only works within allowed directories.",
inputSchema: zodToJsonSchema(EditFileArgsSchema) as ToolInput, inputSchema: zodToJsonSchema(EditFileArgsSchema) as ToolInput,

View file

@ -20,10 +20,12 @@
}, },
"dependencies": { "dependencies": {
"@modelcontextprotocol/sdk": "0.5.0", "@modelcontextprotocol/sdk": "0.5.0",
"diff": "^5.1.0",
"glob": "^10.3.10", "glob": "^10.3.10",
"zod-to-json-schema": "^3.23.5" "zod-to-json-schema": "^3.23.5"
}, },
"devDependencies": { "devDependencies": {
"@types/diff": "^5.0.9",
"@types/node": "^20.11.0", "@types/node": "^20.11.0",
"shx": "^0.3.4", "shx": "^0.3.4",
"typescript": "^5.3.3" "typescript": "^5.3.3"