List the held entries a unit record reads, and leave units that register units or add class fields without one (#2431)

packages/compiler/src/frontend/lowering/class-attributes.ts

@@ -53,13 +53,14 @@ function attributeClassNamed(lowerer: Lowerer, className: string): ClassInfo | n
 
 /** Adds a zero-initialized f64 to every class of the hierarchy, at the same
  * offset, like the property bag. */
-function addHiddenField(info: ClassInfo, name: string): void {
+function addHiddenField(lowerer: Lowerer, info: ClassInfo, name: string): void {
   let root = info;
   while (root.base && !root.base.def.runtime) root = root.base;
   if (root.fields.has(name)) return;
   const index = root.def.fields.length;
   const add = (current: ClassInfo): void => {
     if (!current.fields.has(name)) {
+      lowerer.classLayoutChanges++;
       current.fields.set(name, F64);
       current.def.fields.splice(index, 0, { name, type: F64 });
     }
@@ -78,7 +79,7 @@ export function addUntypedIntegrityWords(lowerer: Lowerer): void {
   if (!lowerer.untypedIntegrity) return;
   for (const [className, info] of lowerer.classes) {
     if (isDynTypedRefType({ kind: "object", className }) && attributeClassNamed(lowerer, className))
-      addHiddenField(info, CLASS_ATTRIBUTES);
+      addHiddenField(lowerer, info, CLASS_ATTRIBUTES);
   }
 }
 
@@ -97,7 +98,7 @@ export function lowerClassIntegrity(
 ): IrExpr | null {
   const info = attributeClass(lowerer, target);
   if (!info) return null;
-  addHiddenField(info, CLASS_ATTRIBUTES);
+  addHiddenField(lowerer, info, CLASS_ATTRIBUTES);
   const loc = locOf(call);
   const type = target.type as ClassObjectType;
   const bits =
@@ -204,8 +205,8 @@ export function lowerClassFieldDefine(
   )
     return null;
   const attributesField = classFieldAttributesName(field);
-  addHiddenField(info, CLASS_ATTRIBUTES);
-  addHiddenField(info, attributesField);
+  addHiddenField(lowerer, info, CLASS_ATTRIBUTES);
+  addHiddenField(lowerer, info, attributesField);
   const loc = locOf(call);
   const type = target.type as ClassObjectType;
   const receiverSlot = lowerer.declareHiddenLocal("%defineReceiver", type);

packages/compiler/src/frontend/lowering/lowerer.ts

@@ -3798,6 +3798,11 @@ export class Lowerer {
   /** The events of the unit being recorded (unit-journal.ts); null
    * otherwise. */
   unitEvents: UnitEvents | null = null;
+  /** Fields added to collected classes (class-attributes.ts) and reachable
+   * units registered while units lower: a unit that changes either has no
+   * record (journalScalars). */
+  classLayoutChanges = 0;
+  unitRegistrations = 0;
 
   classCanBeConstructed(info: ClassInfo): boolean {
     return (
@@ -4440,6 +4445,8 @@ export class Lowerer {
       fixed("onExprClassCollected", () => this.onExprClassCollected),
       fixed("onLateClassCollected", () => this.onLateClassCollected),
       fixed("onDeclarationRegistered", () => this.onDeclarationRegistered),
+      fixed("classLayoutChanges", () => this.classLayoutChanges),
+      fixed("unitRegistrations", () => this.unitRegistrations),
       fixed("liftDynFieldPolicy", () => this.liftDynFieldPolicy),
       fixed("coercionOriginNode", () => this.coercionOriginNode),
       fixed("coercionOriginUnion", () => this.coercionOriginUnion),
@@ -9855,6 +9862,7 @@ export class Lowerer {
         lower: () => IrFunction | null,
       ): void => {
         units.set(name, { order: unitOrder++, roots, lower });
+        this.unitRegistrations++;
         if (reachable.has(name)) queue.push(name);
         metadataPriority.set(name, [3, expressionMetadataOrder++]);
       };
@@ -9895,6 +9903,7 @@ export class Lowerer {
           roots: functionRoots(decl),
           lower: () => this.lowerFunction(decl),
         });
+        this.unitRegistrations++;
         if (reachable.has(sig.name)) queue.push(sig.name);
         metadataPriority.set(sig.name, [3, expressionMetadataOrder++]);
       } else if (ts.isClassDeclaration(decl)) {
@@ -9915,6 +9924,7 @@ export class Lowerer {
             roots: functionRoots(entry.member),
             lower: () => lowerStaticMethod(this, info, name),
           });
+          this.unitRegistrations++;
           if (reachable.has(unit)) queue.push(unit);
           metadataPriority.set(unit, [3, expressionMetadataOrder++]);
         }

packages/compiler/src/frontend/lowering/unit-journal.test.ts

@@ -3,6 +3,8 @@ import type { Program, SourceFile } from "../ts7/adapter.js";
 import { Lowerer } from "./lowerer.js";
 import {
   checkedTable,
+  contentDigest,
+  encodeUnitFunction,
   flagScalar,
   functionTable,
   lowerWaveRecorded,
@@ -342,3 +344,33 @@ test("a unit that changes a function it names in place has no record", () => {
   lowerWaveRecorded(subject, ["a"], stats);
   expect(stats).toMatchObject({ recorded: 0, unrecorded: 1, reasons: { "lifted:changed": 1 } });
 });
+
+test("a record lists the held functions it names, with their digests", () => {
+  const functions = [fn("%shared"), fn("%unused")];
+  const subject: WaveHost = {
+    tables: [functionTable("lifted", functions, (name) => functions.some((f) => f.name === name))],
+    scalars: [],
+    lowerRecorded: () => {
+      functions.push(fn("%own"));
+      return {
+        fn: {
+          ...fn("%a"),
+          body: [
+            { kind: "call", callee: "%shared" },
+            { kind: "call", callee: "%own" },
+          ] as never,
+        },
+        poisoned: false,
+      };
+    },
+    store: () => {},
+    statementCounts: () => ({ total: 0, failed: 0, other: 0 }),
+    resolves: () => true,
+  };
+  const { record } = recordUnit(subject, "a");
+  // The function the unit added is in its entries, not its reads.
+  expect(record!.reads).toEqual({
+    lifted: [["%shared", contentDigest(encodeUnitFunction(functions[0]!)!)]],
+  });
+  expect(Object.keys(record!.tables)).toEqual(["lifted"]);
+});

packages/compiler/src/frontend/lowering/unit-journal.ts

@@ -72,6 +72,10 @@ export interface UnitRecord {
   /** The encoded entries the unit added, by table name, for the tables it
    * grew. */
   readonly tables: Readonly<Record<string, readonly string[]>>;
+  /** The entries the unit's IR and new entries name that its tables held
+   * before it, by table name, each with a digest of its content. Applying
+   * the record needs each to be held with the same content. */
+  readonly reads: Readonly<Record<string, readonly (readonly [string, string])[]>>;
 }
 
 /** A table a unit's lowering can grow. An appendable table only adds
@@ -87,6 +91,9 @@ export interface JournalTable {
   readonly entries?: (from: number) => string[] | null;
   /** Adds the encoded entries whose keys this process's table lacks. */
   readonly apply?: (entries: readonly string[]) => void;
+  /** A digest of the content of the entry under `key`, when the table
+   * holds one before position `before`; null otherwise. */
+  readonly digest?: (key: string, before: number) => string | null;
   /** Removes the entries from position `from` on. */
   readonly truncate?: (from: number) => void;
   /** The entries the table holds now. */
@@ -108,6 +115,18 @@ export interface HeldContents {
   refresh(keys: ReadonlySet<string>): void;
 }
 
+/** A digest of `text`: its length and two 32-bit FNV-1a hashes. */
+export function contentDigest(text: string): string {
+  let a = 0x811c9dc5;
+  let b = 0x01000193 ^ 0x5bd1e995;
+  for (let i = 0; i < text.length; i++) {
+    const c = text.charCodeAt(i);
+    a = Math.imul(a ^ c, 0x01000193);
+    b = Math.imul(b ^ c, 0x01000193) ^ (b >>> 13);
+  }
+  return `${text.length}:${(a >>> 0).toString(16)}${(b >>> 0).toString(16)}`;
+}
+
 /** The strings quoted in JSON `texts`, unescaped only as far as their
  * delimiting quotes: the names a unit's encoded IR and entries refer to. */
 export function quotedStrings(texts: readonly string[]): Set<string> {
@@ -310,18 +329,19 @@ export function recordUnit(
       if (entries === null) reasons.push(`${table.name}:encoding`);
       else added[table.name] = entries;
     }
+  // A unit uses the entries its IR and new entries name: it reads them, and
+  // a unit that changes one in place changes one it names.
+  const texts: string[] = [];
+  if (text !== null) texts.push(text);
+  for (const index of grown) {
+    const table = tables[index]!;
+    const entries = added[table.name] ?? table.entries!(before[index]!);
+    if (entries !== null) for (const entry of entries) texts.push(entry);
+  }
+  const names = quotedStrings(texts);
   if (contents !== null) {
-    // A unit changes in place the entries it uses, which its IR and new
-    // entries name: a unit with a record must leave them as they were, and
-    // a unit without one keeps its changes.
-    const texts: string[] = [];
-    if (text !== null) texts.push(text);
-    for (const index of grown) {
-      const table = tables[index]!;
-      const entries = added[table.name] ?? table.entries!(before[index]!);
-      if (entries !== null) for (const entry of entries) texts.push(entry);
-    }
-    const names = quotedStrings(texts);
+    // A unit with a record must leave the entries it names as they were; a
+    // unit without one keeps its changes.
     if (reasons.length === 0)
       tables.forEach((table, index) => {
         if ((contents[index] ?? null)?.changedAmong(names) != null)
@@ -330,8 +350,24 @@ export function recordUnit(
     if (reasons.length > 0) for (const held of contents) held?.refresh(names);
     for (const held of contents) held?.extend();
   }
+  const reads: Record<string, (readonly [string, string])[]> = {};
+  if (reasons.length === 0)
+    tables.forEach((table, index) => {
+      if (table.digest === undefined) return;
+      const read: (readonly [string, string])[] = [];
+      for (const key of names) {
+        const digest = table.digest(key, before[index]!);
+        if (digest !== null) read.push([key, digest]);
+      }
+      if (read.length > 0) reads[table.name] = read;
+    });
   if (reasons.length > 0) return { record: null, reasons, fn, before };
-  return { record: { name, fn: text, poisoned, events, tables: added }, reasons, fn, before };
+  return {
+    record: { name, fn: text, poisoned, events, tables: added, reads },
+    reasons,
+    fn,
+    before,
+  };
 }
 
 /** Lowers `wave` with every unit recorded. A unit with a record keeps the
@@ -539,9 +575,37 @@ export function functionTable(
   has: (name: string) => boolean,
   changed: () => void = () => {},
 ): JournalTable {
+  // The position of each function by name, for the functions indexed so far.
+  let indexes = new Map<string, number>();
+  let indexed = 0;
+  const lookup = (key: string): number | undefined => {
+    for (; indexed < functions.length; indexed++) indexes.set(functions[indexed]!.name, indexed);
+    const index = indexes.get(key);
+    return index !== undefined && functions[index]?.name === key ? index : undefined;
+  };
+  const indexOf = (key: string): number | undefined => {
+    if (indexed > functions.length) {
+      indexes = new Map();
+      indexed = 0;
+    }
+    const index = lookup(key);
+    if (index !== undefined || !has(key)) return index;
+    // A list that was cut and grown again holds other functions at the
+    // positions indexed before.
+    indexes = new Map();
+    indexed = 0;
+    return lookup(key);
+  };
   return {
     name,
     size: () => functions.length,
+    digest: (key, before) => {
+      if (!has(key)) return null;
+      const index = indexOf(key);
+      if (index === undefined || index >= before) return null;
+      const text = encodeUnitFunction(functions[index]!);
+      return text === null ? null : contentDigest(text);
+    },
     entries: (from) => {
       const out: string[] = [];
       for (const fn of functions.slice(from)) {

packages/compiler/test/recorded-lowering.test.ts

@@ -70,3 +70,36 @@ test("two builds of a program write the same records", async () => {
     await rm(dir, { recursive: true, force: true });
   }
 }, 300_000);
+
+/** The reasons in a program's records that leave units without one. */
+async function recordReasons(program: string, dir: string): Promise<Set<string>> {
+  process.env["SCRIPTC_LOWER_JOURNAL_DUMP"] = join(dir, program);
+  try {
+    await lower(program, dir, "dump");
+  } finally {
+    delete process.env["SCRIPTC_LOWER_JOURNAL_DUMP"];
+  }
+  const reasons = new Set<string>();
+  for (const line of (await readFile(join(dir, program, "records.jsonl"), "utf8"))
+    .trimEnd()
+    .split("\n"))
+    for (const reason of (JSON.parse(line) as { reasons?: string[] }).reasons ?? [])
+      reasons.add(reason);
+  return reasons;
+}
+
+test("units that register reachable units or add class fields have no record", async () => {
+  const dir = await mkdtemp(join(tmpdir(), "scriptc-recorded-lowering-reasons-"));
+  try {
+    // A function-local class registers its members as reachable units.
+    expect(await recordReasons("5945-local-class-observed-methods.ts", dir)).toContain(
+      "unitRegistrations",
+    );
+    // Object.freeze of a class instance adds the class's integrity field.
+    expect(await recordReasons("6004-class-instance-integrity.ts", dir)).toContain(
+      "classLayoutChanges",
+    );
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}, 300_000);