mirror of
https://github.com/vercel-labs/scriptc.git
synced 2026-11-01 07:33:12 +08:00
Chris Tate
db2cb5c99e
readline split redirected input byte by byte, twice per chunk (once to find whether the chunk ends a line, once to split it). Before splitting it copied each chunk out of its Buffer and into a scratch buffer, decoded it into a new string, and copied the unfinished line and the whole chunk into a joined buffer. Now: - one scan per chunk: memchr for a CR once, then LF to LF with memchr when the chunk has none, and the LF, CR and CRLF split otherwise; - the unfinished line joins only the first line of the chunk; - an ASCII chunk with no partial UTF-8 sequence held from the previous one is split where it is, and a chunk that is not shared memory is read from its own storage. Lines are still copied into their strings before the first is delivered. Buffer.from(string, 'latin1' | 'ascii') encoded character by character; an ASCII prefix, all of an ASCII string, is now copied with memcpy. Line events, their order and the CRLF delay are unchanged. Corpus 8344 covers lines joined across chunks, a line longer than a chunk, empty lines, mixed line endings and UTF-8 split across chunks. The new line-reader workload reads a 67 MB log through readline over a file stream.
35 lines
1.4 KiB
TypeScript
35 lines
1.4 KiB
TypeScript
// Line-by-line processing of a log file with readline over a file stream:
|
|
// 'line' events for about 67 MB of access-log lines, with light work per
|
|
// line (a status count, a path tally, byte totals).
|
|
import { createReadStream } from "node:fs";
|
|
import { createInterface } from "node:readline";
|
|
|
|
const path = process.argv[2]!;
|
|
const statuses = new Map<string, number>();
|
|
const paths = new Map<string, number>();
|
|
let lines = 0;
|
|
let chars = 0;
|
|
let bytes = 0;
|
|
let longest = 0;
|
|
|
|
const input = createInterface({ input: createReadStream(path), crlfDelay: Infinity });
|
|
input.on("line", (line: string) => {
|
|
lines++;
|
|
chars += line.length;
|
|
if (line.length > longest) longest = line.length;
|
|
const quote = line.indexOf('"');
|
|
const space = line.indexOf(" ", quote);
|
|
const path = line.slice(space + 1, line.indexOf("/", line.indexOf("/", space) + 1));
|
|
paths.set(path, (paths.get(path) ?? 0) + 1);
|
|
const close = line.indexOf('"', space);
|
|
const status = line.slice(close + 2, close + 5);
|
|
statuses.set(status, (statuses.get(status) ?? 0) + 1);
|
|
bytes += Number(line.slice(close + 6, line.indexOf(" ", close + 6)));
|
|
});
|
|
input.on("close", () => {
|
|
console.log(`${lines} lines, ${chars} characters, longest ${longest}`);
|
|
console.log(`bytes ${bytes}`);
|
|
for (const status of [...statuses.keys()].sort()) console.log(`status ${status}: ${statuses.get(status)}`);
|
|
for (const prefix of [...paths.keys()].sort()) console.log(`path ${prefix}: ${paths.get(prefix)}`);
|
|
});
|