diff --git a/README.md b/README.md index e0dbfc8..d25c3a2 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,9 @@ Reading a file line by line may seem trivial, but in Node.js there's no straight - 📦 **Zero dependencies** — only uses Node.js built-ins - 🔄 **Synchronous** — no callbacks or promises to manage - 💾 **Memory efficient** — reads in chunks, doesn't load entire file -- 🔧 **Configurable** — custom chunk sizes and line endings +- 🔧 **Configurable** — custom chunk sizes - 📘 **TypeScript support** — includes type definitions +- 🪟 **Cross-platform** — handles LF, CRLF, and CR line endings automatically ## 📦 Installation @@ -65,7 +66,6 @@ new LineByLine(fd, [options]) | `filename` | `string` | Path to the file to read | | `fd` | `number` | File descriptor (alternative to filename) | | `options.readChunk` | `number` | Bytes to read at once. Default: `1024` | -| `options.newLineCharacter` | `string` | Line ending character. Default: `\n` | ### Methods @@ -141,14 +141,6 @@ while (line = liner.next()) { } ``` -### Reading Windows-style line endings - -```javascript -const liner = new LineByLine('./windows-file.txt', { - newLineCharacter: '\r\n' -}); -``` - ### Early termination ```javascript @@ -174,6 +166,18 @@ while (line = liner.next()) { - Empty lines are preserved and returned as empty buffers - Returns `null` (not `false`) when end of file is reached +### Line Ending Support + +The library automatically handles all common line ending formats: + +| Format | Characters | Platform | +|--------|------------|----------| +| **LF** | `\n` | Unix, Linux, macOS | +| **CRLF** | `\r\n` | Windows | +| **CR** | `\r` | Classic Mac OS | + +Files with mixed line endings are also supported — each line is detected individually. + ## 📄 License MIT © [Yoan Arnaudov](https://github.com/nacholibre) diff --git a/readlines.d.ts b/readlines.d.ts index f9b9848..0edfa03 100644 --- a/readlines.d.ts +++ b/readlines.d.ts @@ -6,6 +6,7 @@ declare class LineByLine { /** * Creates a new line-by-line file reader. + * Automatically handles LF (\n), CRLF (\r\n), and CR (\r) line endings. * @param file - Path to file or file descriptor * @param options - Configuration options */ @@ -14,8 +15,6 @@ declare class LineByLine { options?: { /** Number of bytes to read at once. Default: 1024 */ readChunk?: number; - /** Line ending character. Default: '\n' */ - newLineCharacter?: string; } ); @@ -37,4 +36,3 @@ declare class LineByLine { } export = LineByLine; - diff --git a/readlines.js b/readlines.js index cf9ffce..bc30134 100644 --- a/readlines.js +++ b/readlines.js @@ -2,6 +2,9 @@ const fs = require('fs'); +const LF = 0x0a; // \n - Unix/Linux/macOS +const CR = 0x0d; // \r - Classic Mac OS / part of Windows CRLF + /** * @class */ @@ -11,12 +14,6 @@ class LineByLine { if (!options.readChunk) options.readChunk = 1024; - if (!options.newLineCharacter) { - options.newLineCharacter = 0x0a; //linux line ending - } else { - options.newLineCharacter = options.newLineCharacter.charCodeAt(0); - } - if (typeof file === 'number') { this.fd = file; } else { @@ -25,29 +22,24 @@ class LineByLine { this.options = options; - this.newLineCharacter = options.newLineCharacter; - this.reset(); } - _searchInBuffer(buffer, hexNeedle) { - let found = -1; - - for (let i = 0; i <= buffer.length; i++) { - let b_byte = buffer[i]; - if (b_byte === hexNeedle) { - found = i; - break; + _searchInBuffer(buffer) { + for (let i = 0; i < buffer.length; i++) { + const byte = buffer[i]; + if (byte === LF || byte === CR) { + return i; } } - - return found; + return -1; } reset() { this.eofReached = false; this.linesCache = []; this.fdPosition = 0; + this.lastChunkEndedWithCR = false; } close() { @@ -55,31 +47,56 @@ class LineByLine { this.fd = null; } - _extractLines(buffer) { - let line; + _extractLines(buffer, isEof) { const lines = []; - let bufferPosition = 0; - - let lastNewLineBufferPosition = 0; - while (true) { - let bufferPositionValue = buffer[bufferPosition++]; - - if (bufferPositionValue === this.newLineCharacter) { - line = buffer.slice(lastNewLineBufferPosition, bufferPosition); - lines.push(line); - lastNewLineBufferPosition = bufferPosition; - } else if (bufferPositionValue === undefined) { - break; + let lineStart = 0; + + // If last chunk ended with CR and this one starts with LF, skip the LF + if (this.lastChunkEndedWithCR && buffer.length > 0 && buffer[0] === LF) { + lineStart = 1; + } + this.lastChunkEndedWithCR = false; + + for (let i = lineStart; i < buffer.length; i++) { + const byte = buffer[i]; + + if (byte === LF) { + // LF found - extract line (without the LF) + lines.push(buffer.slice(lineStart, i)); + lineStart = i + 1; + } else if (byte === CR) { + const lineEnd = i; + + // Check if this is the last byte in the buffer + if (i + 1 >= buffer.length) { + // CR at end of buffer - might be start of CRLF + if (!isEof) { + // Not at EOF, mark that we ended with CR + this.lastChunkEndedWithCR = true; + } + // Extract line without the CR + lines.push(buffer.slice(lineStart, lineEnd)); + lineStart = i + 1; + } else if (buffer[i + 1] === LF) { + // CRLF - skip both characters + lines.push(buffer.slice(lineStart, lineEnd)); + i++; // Skip the LF + lineStart = i + 1; + } else { + // Standalone CR (classic Mac) + lines.push(buffer.slice(lineStart, lineEnd)); + lineStart = i + 1; + } } } - let leftovers = buffer.slice(lastNewLineBufferPosition, bufferPosition); - if (leftovers.length) { - lines.push(leftovers); + // Add any remaining content (incomplete line without newline) + if (lineStart < buffer.length) { + lines.push(buffer.slice(lineStart)); } return lines; - }; + } _readChunk(lineLeftovers) { let totalBytesRead = 0; @@ -95,7 +112,7 @@ class LineByLine { this.fdPosition = this.fdPosition + bytesRead; buffers.push(readBuffer); - } while (bytesRead && this._searchInBuffer(buffers[buffers.length-1], this.options.newLineCharacter) === -1); + } while (bytesRead && this._searchInBuffer(buffers[buffers.length-1]) === -1); let bufferData = Buffer.concat(buffers); @@ -105,10 +122,14 @@ class LineByLine { } if (totalBytesRead) { - this.linesCache = this._extractLines(bufferData); + this.linesCache = this._extractLines(bufferData, this.eofReached); if (lineLeftovers) { - this.linesCache[0] = Buffer.concat([lineLeftovers, this.linesCache[0]]); + if (this.linesCache.length > 0) { + this.linesCache[0] = Buffer.concat([lineLeftovers, this.linesCache[0]]); + } else { + this.linesCache.push(lineLeftovers); + } } } @@ -133,12 +154,12 @@ class LineByLine { if (this.linesCache.length) { line = this.linesCache.shift(); - const lastLineCharacter = line[line.length-1]; - - if (lastLineCharacter !== this.newLineCharacter) { + // Check if this might be an incomplete line (no newline found yet) + // This happens when we read a chunk that doesn't contain a newline + if (!this.eofReached && this.linesCache.length === 0) { bytesRead = this._readChunk(line); - if (bytesRead) { + if (bytesRead && this.linesCache.length) { line = this.linesCache.shift(); } } @@ -148,10 +169,6 @@ class LineByLine { this.close(); } - if (line && line[line.length-1] === this.newLineCharacter) { - line = line.slice(0, line.length-1); - } - return line; } } diff --git a/test/fixtures/.gitattributes b/test/fixtures/.gitattributes new file mode 100644 index 0000000..77ca849 --- /dev/null +++ b/test/fixtures/.gitattributes @@ -0,0 +1,6 @@ +# Preserve line endings for test fixtures - do not auto-convert +crlfFile.txt binary +crlfNoEndingNewline.txt binary +mixedLineEndings.txt binary +crOnlyFile.txt binary + diff --git a/test/fixtures/crOnlyFile.txt b/test/fixtures/crOnlyFile.txt new file mode 100644 index 0000000..03aebbd --- /dev/null +++ b/test/fixtures/crOnlyFile.txt @@ -0,0 +1 @@ +cr1 cr2 cr3 \ No newline at end of file diff --git a/test/fixtures/crlfFile.txt b/test/fixtures/crlfFile.txt new file mode 100644 index 0000000..b87108a --- /dev/null +++ b/test/fixtures/crlfFile.txt @@ -0,0 +1,3 @@ +line1 +line2 +line3 diff --git a/test/fixtures/crlfNoEndingNewline.txt b/test/fixtures/crlfNoEndingNewline.txt new file mode 100644 index 0000000..323138c --- /dev/null +++ b/test/fixtures/crlfNoEndingNewline.txt @@ -0,0 +1,3 @@ +windows1 +windows2 +windows3 \ No newline at end of file diff --git a/test/fixtures/mixedLineEndings.txt b/test/fixtures/mixedLineEndings.txt new file mode 100644 index 0000000..67b8ad6 --- /dev/null +++ b/test/fixtures/mixedLineEndings.txt @@ -0,0 +1,4 @@ +mixed1 +line2 +line3 +line4 diff --git a/test/readlines.test.js b/test/readlines.test.js index 3fdf8d4..5537390 100644 --- a/test/readlines.test.js +++ b/test/readlines.test.js @@ -119,3 +119,97 @@ test('should correctly processes NULL character in lines', () => { assert.strictEqual(liner.fd, null, 'fd is null'); }); + +// ============================================ +// LINE ENDING TESTS (LF, CRLF, CR) +// ============================================ + +test('LF: should read Unix/Linux line endings (\\n)', () => { + const liner = new lineByLine(path.resolve(__dirname, 'fixtures/normalFile.txt')); + + assert.strictEqual(liner.next().toString(), 'google.com', 'line 0'); + assert.strictEqual(liner.next().toString(), 'yahoo.com', 'line 1'); + assert.strictEqual(liner.next().toString(), 'yandex.ru', 'line 2'); + assert.strictEqual(liner.next(), null, 'EOF'); +}); + +test('LF: should handle small chunks with Unix line endings', () => { + const liner = new lineByLine(path.resolve(__dirname, 'fixtures/normalFile.txt'), { + readChunk: 5 // Very small chunk to test boundary conditions + }); + + assert.strictEqual(liner.next().toString(), 'google.com', 'line 0'); + assert.strictEqual(liner.next().toString(), 'yahoo.com', 'line 1'); + assert.strictEqual(liner.next().toString(), 'yandex.ru', 'line 2'); + assert.strictEqual(liner.next(), null, 'EOF'); +}); + +test('CRLF: should read Windows line endings (\\r\\n)', () => { + const liner = new lineByLine(path.resolve(__dirname, 'fixtures/crlfFile.txt')); + + assert.strictEqual(liner.next().toString(), 'line1', 'line 0'); + assert.strictEqual(liner.next().toString(), 'line2', 'line 1'); + assert.strictEqual(liner.next().toString(), 'line3', 'line 2'); + assert.strictEqual(liner.next(), null, 'EOF'); +}); + +test('CRLF: should handle Windows file without trailing newline', () => { + const liner = new lineByLine(path.resolve(__dirname, 'fixtures/crlfNoEndingNewline.txt')); + + assert.strictEqual(liner.next().toString(), 'windows1', 'line 0'); + assert.strictEqual(liner.next().toString(), 'windows2', 'line 1'); + assert.strictEqual(liner.next().toString(), 'windows3', 'line 2'); + assert.strictEqual(liner.next(), null, 'EOF'); +}); + +test('CRLF: should handle small chunks with Windows line endings', () => { + const liner = new lineByLine(path.resolve(__dirname, 'fixtures/crlfFile.txt'), { + readChunk: 5 // Very small chunk to test CRLF boundary conditions + }); + + assert.strictEqual(liner.next().toString(), 'line1', 'line 0'); + assert.strictEqual(liner.next().toString(), 'line2', 'line 1'); + assert.strictEqual(liner.next().toString(), 'line3', 'line 2'); + assert.strictEqual(liner.next(), null, 'EOF'); +}); + +test('CRLF: lines should not contain \\r character', () => { + const liner = new lineByLine(path.resolve(__dirname, 'fixtures/crlfFile.txt')); + + let line; + while (line = liner.next()) { + const str = line.toString(); + assert.ok(!str.includes('\r'), `Line should not contain \\r: "${str}"`); + assert.ok(!str.includes('\n'), `Line should not contain \\n: "${str}"`); + } +}); + +test('CR: should read classic Mac line endings (\\r only)', () => { + const liner = new lineByLine(path.resolve(__dirname, 'fixtures/crOnlyFile.txt')); + + assert.strictEqual(liner.next().toString(), 'cr1', 'line 0'); + assert.strictEqual(liner.next().toString(), 'cr2', 'line 1'); + assert.strictEqual(liner.next().toString(), 'cr3', 'line 2'); + assert.strictEqual(liner.next(), null, 'EOF'); +}); + +test('Mixed: should handle mixed line endings (LF and CRLF)', () => { + const liner = new lineByLine(path.resolve(__dirname, 'fixtures/mixedLineEndings.txt')); + + assert.strictEqual(liner.next().toString(), 'mixed1', 'line 0 (LF)'); + assert.strictEqual(liner.next().toString(), 'line2', 'line 1 (CRLF)'); + assert.strictEqual(liner.next().toString(), 'line3', 'line 2 (CRLF)'); + assert.strictEqual(liner.next().toString(), 'line4', 'line 3 (LF)'); + assert.strictEqual(liner.next(), null, 'EOF'); +}); + +test('Mixed: lines should be clean without any line ending characters', () => { + const liner = new lineByLine(path.resolve(__dirname, 'fixtures/mixedLineEndings.txt')); + + let line; + while (line = liner.next()) { + const str = line.toString(); + assert.ok(!str.includes('\r'), `Line should not contain \\r: "${str}"`); + assert.ok(!str.includes('\n'), `Line should not contain \\n: "${str}"`); + } +});