fix(filesystem): preserve UTF-8 across head and tail chunks (#4667)
This commit is contained in:
parent
cf05b37694
commit
11845d51cb
2 changed files with 65 additions and 27 deletions
|
|
@ -27,6 +27,23 @@ import {
|
||||||
vi.mock('fs/promises');
|
vi.mock('fs/promises');
|
||||||
const mockFs = fs as any;
|
const mockFs = fs as any;
|
||||||
|
|
||||||
|
function createMockFileHandle(content: Buffer) {
|
||||||
|
return {
|
||||||
|
read: vi.fn(
|
||||||
|
async (buffer: Buffer, offset: number, length: number, position: number) => {
|
||||||
|
const bytesRead = content.copy(
|
||||||
|
buffer,
|
||||||
|
offset,
|
||||||
|
position,
|
||||||
|
Math.min(position + length, content.length),
|
||||||
|
);
|
||||||
|
return { bytesRead, buffer };
|
||||||
|
},
|
||||||
|
),
|
||||||
|
close: vi.fn().mockResolvedValue(undefined),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
describe('Lib Functions', () => {
|
describe('Lib Functions', () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
|
|
@ -722,6 +739,23 @@ describe('Lib Functions', () => {
|
||||||
expect(mockFileHandle.close).toHaveBeenCalled();
|
expect(mockFileHandle.close).toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('preserves UTF-8 characters split across chunk boundaries', async () => {
|
||||||
|
const content = Buffer.concat([
|
||||||
|
Buffer.from('discard\n'),
|
||||||
|
Buffer.from('界'),
|
||||||
|
Buffer.alloc(1017, 'a'),
|
||||||
|
Buffer.from('\nlast'),
|
||||||
|
]);
|
||||||
|
const mockFileHandle = createMockFileHandle(content);
|
||||||
|
|
||||||
|
mockFs.stat.mockResolvedValue({ size: content.length } as any);
|
||||||
|
mockFs.open.mockResolvedValue(mockFileHandle);
|
||||||
|
|
||||||
|
const result = await tailFile('/test/file.txt', 2);
|
||||||
|
|
||||||
|
expect(result).toBe(`界${'a'.repeat(1017)}\nlast`);
|
||||||
|
});
|
||||||
|
|
||||||
it('handles read errors gracefully', async () => {
|
it('handles read errors gracefully', async () => {
|
||||||
mockFs.stat.mockResolvedValue({ size: 100 } as any);
|
mockFs.stat.mockResolvedValue({ size: 100 } as any);
|
||||||
|
|
||||||
|
|
@ -778,6 +812,20 @@ describe('Lib Functions', () => {
|
||||||
expect(mockFileHandle.close).toHaveBeenCalled();
|
expect(mockFileHandle.close).toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('preserves UTF-8 characters split across chunk boundaries', async () => {
|
||||||
|
const content = Buffer.concat([
|
||||||
|
Buffer.alloc(1023, 'a'),
|
||||||
|
Buffer.from('界\nsecond'),
|
||||||
|
]);
|
||||||
|
const mockFileHandle = createMockFileHandle(content);
|
||||||
|
|
||||||
|
mockFs.open.mockResolvedValue(mockFileHandle);
|
||||||
|
|
||||||
|
const result = await headFile('/test/file.txt', 1);
|
||||||
|
|
||||||
|
expect(result).toBe(`${'a'.repeat(1023)}界`);
|
||||||
|
});
|
||||||
|
|
||||||
it('handles files with leftover content', async () => {
|
it('handles files with leftover content', async () => {
|
||||||
const mockFileHandle = {
|
const mockFileHandle = {
|
||||||
read: vi.fn(),
|
read: vi.fn(),
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import fs from "fs/promises";
|
||||||
import path from "path";
|
import path from "path";
|
||||||
import os from 'os';
|
import os from 'os';
|
||||||
import { randomBytes } from 'crypto';
|
import { randomBytes } from 'crypto';
|
||||||
|
import { StringDecoder } from 'string_decoder';
|
||||||
import { diffLines, createTwoFilesPatch } from 'diff';
|
import { diffLines, createTwoFilesPatch } from 'diff';
|
||||||
import { minimatch } from 'minimatch';
|
import { minimatch } from 'minimatch';
|
||||||
import { normalizePath, expandHome } from './path-utils.js';
|
import { normalizePath, expandHome } from './path-utils.js';
|
||||||
|
|
@ -364,42 +365,28 @@ export async function tailFile(filePath: string, numLines: number): Promise<stri
|
||||||
// Open file for reading
|
// Open file for reading
|
||||||
const fileHandle = await fs.open(filePath, 'r');
|
const fileHandle = await fs.open(filePath, 'r');
|
||||||
try {
|
try {
|
||||||
const lines: string[] = [];
|
const chunks: Buffer[] = [];
|
||||||
let position = fileSize;
|
let position = fileSize;
|
||||||
let chunk = Buffer.alloc(CHUNK_SIZE);
|
const chunk = Buffer.alloc(CHUNK_SIZE);
|
||||||
let linesFound = 0;
|
let newlinesFound = 0;
|
||||||
let remainingText = '';
|
|
||||||
|
|
||||||
// Read chunks from the end of the file until we have enough lines
|
// Read chunks from the end of the file until we have enough lines
|
||||||
while (position > 0 && linesFound < numLines) {
|
while (position > 0 && newlinesFound < numLines) {
|
||||||
const size = Math.min(CHUNK_SIZE, position);
|
const size = Math.min(CHUNK_SIZE, position);
|
||||||
position -= size;
|
position -= size;
|
||||||
|
|
||||||
const { bytesRead } = await fileHandle.read(chunk, 0, size, position);
|
const { bytesRead } = await fileHandle.read(chunk, 0, size, position);
|
||||||
if (!bytesRead) break;
|
if (!bytesRead) break;
|
||||||
|
|
||||||
// Get the chunk as a string and prepend any remaining text from previous iteration
|
const readData = Buffer.from(chunk.subarray(0, bytesRead));
|
||||||
const readData = chunk.slice(0, bytesRead).toString('utf-8');
|
chunks.unshift(readData);
|
||||||
const chunkText = readData + remainingText;
|
for (const byte of readData) {
|
||||||
|
if (byte === 0x0a) newlinesFound++;
|
||||||
// Split by newlines and count
|
|
||||||
const chunkLines = normalizeLineEndings(chunkText).split('\n');
|
|
||||||
|
|
||||||
// If this isn't the end of the file, the first line is likely incomplete
|
|
||||||
// Save it to prepend to the next chunk
|
|
||||||
if (position > 0) {
|
|
||||||
remainingText = chunkLines[0];
|
|
||||||
chunkLines.shift(); // Remove the first (incomplete) line
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add lines to our result (up to the number we need)
|
|
||||||
for (let i = chunkLines.length - 1; i >= 0 && linesFound < numLines; i--) {
|
|
||||||
lines.unshift(chunkLines[i]);
|
|
||||||
linesFound++;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return lines.join('\n');
|
const text = normalizeLineEndings(Buffer.concat(chunks).toString('utf-8'));
|
||||||
|
return text.split('\n').slice(-numLines).join('\n');
|
||||||
} finally {
|
} finally {
|
||||||
await fileHandle.close();
|
await fileHandle.close();
|
||||||
}
|
}
|
||||||
|
|
@ -413,13 +400,14 @@ export async function headFile(filePath: string, numLines: number): Promise<stri
|
||||||
let buffer = '';
|
let buffer = '';
|
||||||
let bytesRead = 0;
|
let bytesRead = 0;
|
||||||
const chunk = Buffer.alloc(1024); // 1KB buffer
|
const chunk = Buffer.alloc(1024); // 1KB buffer
|
||||||
|
const decoder = new StringDecoder('utf-8');
|
||||||
|
|
||||||
// Read chunks and count lines until we have enough or reach EOF
|
// Read chunks and count lines until we have enough or reach EOF
|
||||||
while (lines.length < numLines) {
|
while (lines.length < numLines) {
|
||||||
const result = await fileHandle.read(chunk, 0, chunk.length, bytesRead);
|
const result = await fileHandle.read(chunk, 0, chunk.length, bytesRead);
|
||||||
if (result.bytesRead === 0) break; // End of file
|
if (result.bytesRead === 0) break; // End of file
|
||||||
bytesRead += result.bytesRead;
|
bytesRead += result.bytesRead;
|
||||||
buffer += chunk.slice(0, result.bytesRead).toString('utf-8');
|
buffer += decoder.write(chunk.subarray(0, result.bytesRead));
|
||||||
|
|
||||||
const newLineIndex = buffer.lastIndexOf('\n');
|
const newLineIndex = buffer.lastIndexOf('\n');
|
||||||
if (newLineIndex !== -1) {
|
if (newLineIndex !== -1) {
|
||||||
|
|
@ -431,6 +419,8 @@ export async function headFile(filePath: string, numLines: number): Promise<stri
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
buffer += decoder.end();
|
||||||
|
|
||||||
// If there is leftover content and we still need lines, add it
|
// If there is leftover content and we still need lines, add it
|
||||||
if (buffer.length > 0 && lines.length < numLines) {
|
if (buffer.length > 0 && lines.length < numLines) {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue