trending/codebase-memory-mcp
1
0
Fork 0
mirror of https://github.com/DeusData/codebase-memory-mcp.git synced 2026-10-21 05:59:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2551 from DeusData/feat/doc-links-documents

Browse Source 
feat(doc-links): Markdown, ADR, reST, AsciiDoc and PDF sections link to the code they name
This commit is contained in:
Martin Vogel
2026-10-10 09:35:07 +02:00
committed by GitHub
parent ec6f837296 cb24c41f3b
commit cb8b641f70
57 changed files with 28357 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile.cbm
+16 -1
View File
@@ -375,6 +375,17 @@ EXTRACTION_SRCS = \
$(CBM_DIR)/extract_dbt.c \
$(CBM_DIR)/doclink.c \
$(CBM_DIR)/doclink_cs.c \
$(CBM_DIR)/doclink_md.c \
$(CBM_DIR)/doclink_rst.c \
$(CBM_DIR)/doclink_py.c \
$(CBM_DIR)/doc_adr.c \
$(CBM_DIR)/doc_pdf.c \
$(CBM_DIR)/doc_adoc.c \
$(CBM_DIR)/pdf/pdf_obj.c \
$(CBM_DIR)/pdf/pdf_font.c \
$(CBM_DIR)/pdf/pdf_text.c \
$(CBM_DIR)/pdf/pdf_tables.c \
$(CBM_DIR)/pdf/pdf_unicode.c \
$(CBM_DIR)/sql_values.c \
$(CBM_DIR)/helpers.c \
$(CBM_DIR)/cpp_branch_views.c \
@@ -497,6 +508,10 @@ PIPELINE_SRCS = \
src/pipeline/pass_doclinks.c \
src/pipeline/doc_links.c \
src/pipeline/doc_links_cs.c \
src/pipeline/doc_links_md.c \
src/pipeline/doc_links_pdf.c \
src/pipeline/doc_links_rst.c \
src/pipeline/doc_links_adoc.c \
src/pipeline/doc_links_msbuild.c \
src/pipeline/pass_route_nodes.c \
src/pipeline/pass_enrichment.c \
@@ -759,7 +774,7 @@ TEST_DISCOVER_SRCS = \
TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c
TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c
TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c
TEST_WATCHER_SRCS = tests/test_watcher.c
THIRD_PARTY.md
+103
View File
@@ -138,6 +138,109 @@ Semantic vector search uses static token embeddings derived from the
See `vendored/nomic/NOTICE` for the exact derivation procedure
(per-token inference + int8 quantization via `scripts/extract_nomic_vectors.py`).
## Embedded Character Data (PDF text layer)
The PDF text-layer extractor (`internal/cbm/pdf/`) carries static tables
derived from the following sources. No upstream source code is included.
- **Adobe Glyph List** — the glyph-name subset in `internal/cbm/pdf/pdf_tables.c`
(the names of the standard Latin encodings and common symbols, mapped to
Unicode) follows [adobe-type-tools/agl-aglfn](https://github.com/adobe-type-tools/agl-aglfn)
`glyphlist.txt`.
- **License:** BSD-3-Clause
- **Copyright:** Copyright 2002-2019 Adobe (http://www.adobe.com/).
- **Adobe core-14 font metrics** — the advance widths of the printable ASCII
range of Helvetica and Times-Roman in `pdf_tables.c` (used only for a
standard-14 font that omits its `/Widths`) are the values of Adobe's
core-14 AFM files. The AFM files themselves are not distributed.
- **Copyright:** Copyright (c) 1985, 1987, 1989, 1990, 1997 Adobe Systems
Incorporated. All Rights Reserved.
- **Unicode Character Database** — `internal/cbm/pdf/pdf_unicode.c` (NFKC of
each code point, canonical compositions, character classes) is generated
by `scripts/gen-pdf-unicode.py` from CPython's `unicodedata` (Unicode
16.0.0).
- **License:** Unicode License v3 (text below)
- **Copyright:** Copyright © 1991-2026 Unicode, Inc.
- The base encodings (Standard, WinAnsi, MacRoman, Symbol) are those of
ISO 32000-1, Annex D.
Adobe Glyph List license (BSD-3-Clause):
```
Copyright 2002-2019 Adobe (http://www.adobe.com/).
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
Neither the name of Adobe nor the names of its contributors may be
used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
```
Unicode License v3:
```
UNICODE LICENSE V3
COPYRIGHT AND PERMISSION NOTICE
Copyright © 1991-2026 Unicode, Inc.
NOTICE TO USER: Carefully read the following legal agreement. BY
DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR
SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE
TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT
DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE.
Permission is hereby granted, free of charge, to any person obtaining a
copy of data files and any associated documentation (the "Data Files") or
software and any associated documentation (the "Software") to deal in the
Data Files or Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, and/or sell
copies of the Data Files or Software, and to permit persons to whom the
Data Files or Software are furnished to do so, provided that either (a)
this copyright and permission notice appear with all copies of the Data
Files or Software, or (b) this copyright and permission notice appear in
associated Documentation.
THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF
THIRD PARTY RIGHTS.
IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE
BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES,
OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA
FILES OR SOFTWARE.
Except as contained in this notice, the name of a copyright holder shall
not be used in advertising or otherwise to promote the sale, use or other
dealings in these Data Files or Software without prior written
authorization of the copyright holder.
```
## Hybrid LSP — Reference Language Servers
The Hybrid LSP layer (`internal/cbm/lsp/`) is an original C implementation
internal/cbm/cbm.c
+39
View File
@@ -2802,6 +2802,45 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
cbm_test_fault_inject(rel_path);
#endif
/* A PDF has no grammar: its text layer is read directly, a Section per
* page plus the pages' code mentions (doc_pdf.c). */
if (language == CBM_LANG_PDF) {
result->module_qn = cbm_fqn_module_source_lang(a, project, rel_path, language);
CBMExtractCtx pdf_ctx = {
.arena = a,
.scratch = scratch,
.result = result,
.source = source,
.source_len = source_len,
.language = language,
.project = project,
.rel_path = rel_path,
.module_qn = result->module_qn,
};
cbm_pdf_extract_document(&pdf_ctx);
cbm_index_mark_done(rel_path);
return result;
}
/* AsciiDoc has no grammar either: its headings, includes, attribute
* references and monospace spans are read from the text (doc_adoc.c). */
if (language == CBM_LANG_ASCIIDOC) {
result->module_qn = cbm_fqn_module_source_lang(a, project, rel_path, language);
CBMExtractCtx adoc_ctx = {
.arena = a,
.scratch = scratch,
.result = result,
.source = source,
.source_len = source_len,
.language = language,
.project = project,
.rel_path = rel_path,
.module_qn = result->module_qn,
};
cbm_adoc_extract_document(&adoc_ctx);
cbm_index_mark_done(rel_path);
return result;
}
// Get language spec
const CBMLangSpec *spec = cbm_lang_spec(language);
if (!spec) {
internal/cbm/cbm.h
+8
View File
@@ -209,6 +209,8 @@ typedef enum {
CBM_LANG_ARKTS, // ArkTS (HarmonyOS/OpenHarmony .ets — TypeScript superset + ArkUI)
CBM_LANG_PLSQL, // Oracle PL/SQL
CBM_LANG_CHIALISP, // Chialisp (.clsp/.clib/.clinc — Chia smart-coin s-expression language)
CBM_LANG_PDF, // PDF documents: the text layer, read without a grammar (doc_pdf.c)
CBM_LANG_ASCIIDOC, // AsciiDoc documents, read without a grammar (doc_adoc.c)
CBM_LANG_COUNT
} CBMLanguage;
@@ -292,6 +294,12 @@ typedef struct {
* are bound to the owning result's source identity before cross-file use. */
uint32_t test_name_start_byte, test_name_end_byte;
uint32_t test_body_start_byte, test_body_end_byte;
/* Documents only: facts a document definition adds to its node's
* properties, as a NULL-terminated key, value, key, value, ... list (an
* ADR's id, status, date, deciders, title). Keys are fixed literals; the
* values are escaped where the properties are written. Tail field:
* zero-init stays valid. */
const char **extra_props;
} CBMDefinition;
/* Argument captured from a call expression */
internal/cbm/doc_adoc.c
+944
View File
@@ -0,0 +1,944 @@
/*
* doc_adoc.c — AsciiDoc documents in the graph (the extraction half;
* src/pipeline/doc_links_adoc.c resolves), and the Antora scope blob.
*
* AsciiDoc has no grammar here: like a PDF, the file is read directly. Each
* heading (`= Title` .. `====== Title`, or `#` .. `######`) outside a
* delimited block becomes a Section (name the title, the text up to the next
* heading as its docstring, its lines up to the next heading); text before
* the first heading belongs to the file. Delimited blocks (`----` listing,
* `....` literal, `////` comment, `++++` passthrough, `====` example, `****`
* sidebar, `____` quote, `|===` table, ``` fences) open and close on the same
* delimiter line (the field test's H8 rules).
*
* References become tokens whose `raw` is the reference as written, then
* TAB-separated fields:
* include include::target[attrs] (Asciidoctor's preprocessor directive:
* read in every block but a comment block; `\include::` is
* escaped): written, target with the page's attributes
* substituted (references the page does not define stay as
* `{name}` for the component's), tags (`tag=` / `tags=`),
* lines (`lines=`)
* attribute an attribute reference `{name}` in the text: written, name,
* the page's value as written and as the page expands it (both
* "" when the page does not define it).
* Antora components define their API links as attributes
* (`{Assertions}` -> a javadoc page): the resolver reads them.
* code_path, a monospace span (`x`, ``x``, `+x+`) naming a path or a
* code_name qualified name (the Markdown code-span classifier); raw is
* the span's text
* Page attributes (`:name: value`, `:name!:` / `:!name:` to unset) apply from
* their line on, in document order.
*
* The Antora scope (cbm_doclink_antora_scan_scope): an antora.yml's component
* name, its asciidoc attributes and its collector scans; an
* antora-playbook*.yml's asciidoc attributes. Read as text, never run.
*/
#include "doclink.h"
#include "arena.h"
#include "cbm.h"
#include "helpers.h"
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
enum {
AD_HEAD_MAX = 6, /* `======` */
AD_DELIM_MIN = 4, /* `----` and its kin */
AD_TABLE_MIN = 3, /* `|===` */
AD_FENCE_MIN = 3, /* ``` */
AD_BODY_MAX = 500, /* a section's docstring (Markdown's MAX_COMMENT_LEN) */
AD_SPAN_MAX = 150, /* a span the classifier reads (MD_SPAN_MAX) */
AD_ATTRS_INIT = 16, /* first capacity of the page attribute list */
AD_EXPAND_DEPTH = 10, /* nested attribute references expanded (H8) */
AD_FIELD_MAX = 512, /* an attribute value or target longer than this is skipped */
};
typedef struct {
const char *s;
int len;
} ad_line_t;
typedef struct {
char *name;
char *value;
} ad_attr_t;
typedef struct {
int line; /* 0-based */
const char *title;
const char *qn;
bool shared_qn;
} ad_head_t;
typedef struct {
CBMExtractCtx *ctx;
CBMArena *scratch;
ad_line_t *L;
int n;
unsigned char *in_block; /* 1: inside a delimited block; 2: inside a comment block */
ad_head_t *heads;
int nheads;
ad_attr_t *attrs;
int nattrs;
int cap_attrs;
char *span_buf;
bool failed;
} ad_doc_t;
static bool ad_blank(char c) {
return c == ' ' || c == '\t' || c == '\r';
}
static bool ad_word(char c) {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
}
/* The line without trailing whitespace: its length. */
static int ad_rlen(const ad_line_t *l) {
int n = l->len;
while (n > 0 && ad_blank(l->s[n - 1])) {
n--;
}
return n;
}
/* ADOC_DELIM_RE: the delimiter's length, or 0. */
static int ad_delim(const ad_line_t *l) {
int n = ad_rlen(l);
if (n == 0) {
return 0;
}
const char *s = l->s;
if (s[0] == '|') {
int k = 1;
while (k < n && s[k] == '=') {
k++;
}
return (k == n && n - 1 >= AD_TABLE_MIN) ? n : 0;
}
if (!strchr("-.=*_+/`", s[0])) {
return 0;
}
for (int k = 1; k < n; k++) {
if (s[k] != s[0]) {
return 0;
}
}
int min = s[0] == '`' ? AD_FENCE_MIN : AD_DELIM_MIN;
return n >= min ? n : 0;
}
/* ADOC_HEAD_RE ^(={1,6}|#{1,6})\s+(\S.*?)\s*$: the level and the title span. */
static int ad_heading(const ad_line_t *l, int *ta, int *tb) {
const char *s = l->s;
int n = ad_rlen(l);
if (n == 0 || (s[0] != '=' && s[0] != '#')) {
return 0;
}
int k = 0;
while (k < n && s[k] == s[0]) {
k++;
}
if (k > AD_HEAD_MAX || k >= n || !ad_blank(s[k])) {
return 0;
}
int a = k;
while (a < n && ad_blank(s[a])) {
a++;
}
if (a >= n) {
return 0;
}
*ta = a;
*tb = n;
return k;
}
static bool ad_lines(ad_doc_t *d, const char *src, int len) {
int count = 1;
for (int i = 0; i < len; i++) {
count += src[i] == '\n';
}
d->L = (ad_line_t *)cbm_arena_alloc(d->scratch, (size_t)count * sizeof(*d->L));
d->in_block = (unsigned char *)cbm_arena_alloc(d->scratch, (size_t)count);
if (!d->L || !d->in_block) {
return false;
}
memset(d->in_block, 0, (size_t)count);
int pos = 0;
for (int i = 0; i < count; i++) {
const char *nl = memchr(src + pos, '\n', (size_t)(len - pos));
int end = nl ? (int)(nl - src) : len;
d->L[i] = (ad_line_t){src + pos, end - pos};
pos = end + 1;
}
d->n = count;
return true;
}
/* Delimited blocks, then the headings outside them. */
static void ad_structure(ad_doc_t *d) {
int open = -1;
for (int i = 0; i < d->n; i++) {
int dl = ad_delim(&d->L[i]);
if (open < 0) {
if (dl) {
open = i;
d->in_block[i] = d->L[i].s[0] == '/' ? 2 : 1;
}
continue;
}
const ad_line_t *o = &d->L[open];
d->in_block[i] = d->L[open].s[0] == '/' ? 2 : 1;
if (dl && dl == ad_rlen(o) && memcmp(d->L[i].s, o->s, (size_t)dl) == 0) {
open = -1;
}
}
d->heads = (ad_head_t *)cbm_arena_alloc(d->scratch, (size_t)d->n * sizeof(*d->heads));
if (!d->heads) {
d->failed = true;
return;
}
for (int i = 0; i < d->n; i++) {
int ta;
int tb;
if (d->in_block[i] || ad_heading(&d->L[i], &ta, &tb) == 0) {
continue;
}
char *title = cbm_arena_strndup(d->ctx->arena, d->L[i].s + ta, (size_t)(tb - ta));
if (!title) {
d->failed = true;
return;
}
d->heads[d->nheads++] = (ad_head_t){.line = i, .title = title};
}
}
/* qn_safe_segment (extract_defs.c): whitespace runs become '-'. */
static const char *ad_qn_segment(CBMArena *a, const char *name) {
char *out = cbm_arena_strdup(a, name);
if (!out) {
return NULL;
}
size_t w = 0;
bool in_ws = false;
for (const char *p = name; *p; p++) {
if (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') {
in_ws = true;
continue;
}
if (in_ws && w > 0) {
out[w++] = '-';
}
in_ws = false;
out[w++] = *p;
}
out[w] = '\0';
return out;
}
/* The section's own text, whitespace collapsed, at most AD_BODY_MAX bytes and
* never a cut UTF-8 character. */
static const char *ad_body(ad_doc_t *d, int from, int to) {
char *out = (char *)cbm_arena_alloc(d->ctx->arena, AD_BODY_MAX + 1);
if (!out) {
return NULL;
}
int w = 0;
for (int k = from; k < to && w < AD_BODY_MAX; k++) {
const char *s = d->L[k].s;
int len = d->L[k].len;
for (int i = 0; i < len && w < AD_BODY_MAX; i++) {
char c = s[i];
if (ad_blank(c)) {
if (w > 0 && out[w - 1] != ' ') {
out[w++] = ' ';
}
continue;
}
out[w++] = c;
}
if (w > 0 && w < AD_BODY_MAX && out[w - 1] != ' ') {
out[w++] = ' ';
}
}
while (w > 0 && ((unsigned char)out[w - 1] & 0xC0) == 0x80) {
w--; /* never end inside a character */
}
if (w > 0 && ((unsigned char)out[w - 1] & 0xC0) == 0xC0) {
w--;
}
while (w > 0 && out[w - 1] == ' ') {
w--;
}
out[w] = '\0';
return w > 0 ? out : NULL;
}
static int ad_head_qn_cmp(const void *a, const void *b) {
const ad_head_t *x = *(const ad_head_t *const *)a;
const ad_head_t *y = *(const ad_head_t *const *)b;
return strcmp(x->qn, y->qn);
}
static void ad_sections(ad_doc_t *d) {
CBMExtractCtx *ctx = d->ctx;
CBMArena *a = ctx->arena;
for (int h = 0; h < d->nheads; h++) {
const char *seg = ad_qn_segment(a, d->heads[h].title);
d->heads[h].qn = seg ? cbm_fqn_compute(a, ctx->project, ctx->rel_path, seg) : NULL;
if (!d->heads[h].qn) {
d->failed = true;
return;
}
}
if (d->nheads > 1) {
ad_head_t **by = (ad_head_t **)cbm_arena_alloc(d->scratch, (size_t)d->nheads * sizeof(*by));
if (!by) {
d->failed = true;
return;
}
for (int h = 0; h < d->nheads; h++) {
by[h] = &d->heads[h];
}
qsort(by, (size_t)d->nheads, sizeof(*by), ad_head_qn_cmp);
for (int h = 1; h < d->nheads; h++) {
if (strcmp(by[h]->qn, by[h - 1]->qn) == 0) {
by[h]->shared_qn = true;
by[h - 1]->shared_qn = true;
}
}
}
for (int h = 0; h < d->nheads; h++) {
const ad_head_t *hd = &d->heads[h];
int end = h + 1 < d->nheads ? d->heads[h + 1].line : d->n;
int last = end - 1;
while (last > hd->line && ad_rlen(&d->L[last]) == 0) {
last--;
}
CBMDefinition def;
memset(&def, 0, sizeof(def));
def.name = hd->title;
def.qualified_name = hd->qn;
def.label = "Section";
def.file_path = ctx->rel_path;
def.start_line = (uint32_t)hd->line + 1;
def.end_line = (uint32_t)last + 1;
def.is_exported = true;
def.docstring = ad_body(d, hd->line + 1, end);
cbm_defs_push(&ctx->result->defs, a, def);
}
}
/* ── Page attributes ─────────────────────────────────────────────── */
static ad_attr_t *ad_attr_find(ad_doc_t *d, const char *name, size_t n) {
for (int i = 0; i < d->nattrs; i++) {
if (strlen(d->attrs[i].name) == n && memcmp(d->attrs[i].name, name, n) == 0) {
return &d->attrs[i];
}
}
return NULL;
}
/* ATTR_ENTRY_RE ^:(!?)([\w][\w\-]*)(!?):(?:\s+(.*?))?\s*$ -- true when the
* line is an attribute entry (set or unset). */
static bool ad_attr_entry(ad_doc_t *d, const ad_line_t *l) {
const char *s = l->s;
int n = ad_rlen(l);
if (n < 3 || s[0] != ':') {
return false;
}
int k = 1;
bool unset = false;
if (s[k] == '!') {
unset = true;
k++;
}
int na = k;
if (k >= n || !ad_word(s[k])) {
return false;
}
while (k < n && (ad_word(s[k]) || s[k] == '-')) {
k++;
}
int nb = k;
if (k < n && s[k] == '!') {
unset = true;
k++;
}
if (k >= n || s[k] != ':') {
return false;
}
k++;
if (k < n && !ad_blank(s[k])) {
return false;
}
while (k < n && ad_blank(s[k])) {
k++;
}
ad_attr_t *a = ad_attr_find(d, s + na, (size_t)(nb - na));
if (unset) {
if (a) {
*a = d->attrs[--d->nattrs];
}
return true;
}
char *value = cbm_arena_strndup(d->scratch, s + k, (size_t)(n - k));
if (!value) {
d->failed = true;
return true;
}
if (a) {
a->value = value;
return true;
}
if (d->nattrs >= d->cap_attrs) {
int ncap = d->cap_attrs ? d->cap_attrs * 2 : AD_ATTRS_INIT;
ad_attr_t *grown = (ad_attr_t *)cbm_arena_alloc(d->scratch, (size_t)ncap * sizeof(*grown));
if (!grown) {
d->failed = true;
return true;
}
if (d->nattrs) {
memcpy(grown, d->attrs, (size_t)d->nattrs * sizeof(*grown));
}
d->attrs = grown;
d->cap_attrs = ncap;
}
char *name = cbm_arena_strndup(d->scratch, s + na, (size_t)(nb - na));
if (!name) {
d->failed = true;
return true;
}
d->attrs[d->nattrs++] = (ad_attr_t){name, value};
return true;
}
/* A `{name}` reference at s[i] (not escaped `\{`): the name's span. */
static bool ad_attr_ref(const char *s, int n, int i, int *a, int *b) {
if (s[i] != '{' || (i > 0 && s[i - 1] == '\\') || i + 2 >= n || !ad_word(s[i + 1])) {
return false;
}
int k = i + 1;
while (k < n && (ad_word(s[k]) || s[k] == '-')) {
k++;
}
if (k >= n || s[k] != '}') {
return false;
}
*a = i + 1;
*b = k;
return true;
}
/* The text with the page's attribute references substituted, pass after pass
* (a value may reference others; at most AD_EXPAND_DEPTH passes, as H8's
* depth); references the page does not define stay as written. NULL when it
* grows past AD_FIELD_MAX or memory ran out. */
static char *ad_expand(ad_doc_t *d, const char *s, int n) {
char cur[AD_FIELD_MAX + 1];
char out[AD_FIELD_MAX + 1];
if (n > AD_FIELD_MAX) {
return NULL;
}
memcpy(cur, s, (size_t)n);
int cl = n;
for (int pass = 0; pass < AD_EXPAND_DEPTH; pass++) {
int w = 0;
bool changed = false;
for (int i = 0; i < cl; i++) {
int a;
int b;
const ad_attr_t *at = NULL;
if (ad_attr_ref(cur, cl, i, &a, &b)) {
at = ad_attr_find(d, cur + a, (size_t)(b - a));
}
if (at) {
int vl = (int)strlen(at->value);
if (w + vl > AD_FIELD_MAX) {
return NULL;
}
memcpy(out + w, at->value, (size_t)vl);
w += vl;
i = b;
changed = true;
continue;
}
if (w + 1 > AD_FIELD_MAX) {
return NULL;
}
out[w++] = cur[i];
}
if (w > 0) {
memcpy(cur, out, (size_t)w);
}
cl = w;
if (!changed) {
break;
}
}
return cbm_arena_strndup(d->scratch, cur, (size_t)cl);
}
/* ── Tokens ──────────────────────────────────────────────────────── */
static void ad_push(ad_doc_t *d, int line, int syntax, const char *raw) {
if (!raw) {
d->failed = true;
return;
}
CBMExtractCtx *ctx = d->ctx;
int lo = 0;
int hi = d->nheads;
while (lo < hi) {
int mid = lo + ((hi - lo) / 2);
if (d->heads[mid].line <= line) {
lo = mid + 1;
} else {
hi = mid;
}
}
const ad_head_t *sec = lo > 0 ? &d->heads[lo - 1] : NULL;
if (sec && sec->shared_qn) {
sec = NULL;
}
CBMDocLink link = {
.source_qn = sec ? sec->qn : (ctx->module_qn ? ctx->module_qn : ""),
.raw = raw,
.line = (uint32_t)line + 1,
.def_line = sec ? (uint32_t)sec->line + 1 : 1,
.syntax = (uint16_t)syntax,
.flags = sec ? 0 : CBM_DOCLINK_FLAG_FILE,
};
cbm_doclinks_push(&ctx->result->doc_links, ctx->arena, link);
}
/* A value of the include's attribute list (`tag=x`, `tags="a;b"`, `lines=1..5`). */
static void ad_attrlist_value(const char *al, int n, const char *key, char *out, size_t cap) {
out[0] = '\0';
size_t kl = strlen(key);
for (int i = 0; i + (int)kl < n; i++) {
if ((i > 0 && ad_word(al[i - 1])) || memcmp(al + i, key, kl) != 0) {
continue;
}
int k = i + (int)kl;
while (k < n && ad_blank(al[k])) {
k++;
}
if (k >= n || al[k] != '=') {
continue;
}
k++;
while (k < n && ad_blank(al[k])) {
k++;
}
int a = k;
int b;
if (k < n && al[k] == '"') {
a = k + 1;
const char *q = memchr(al + a, '"', (size_t)(n - a));
b = q ? (int)(q - al) : n;
} else {
b = a;
while (b < n && al[b] != ',') {
b++;
}
}
while (b > a && ad_blank(al[b - 1])) {
b--;
}
snprintf(out, cap, "%.*s", b - a, al + a);
return;
}
}
/* INCLUDE_RE ^(\\?)include::([^\[\s][^\[]*)\[(.*)\]\s*$ */
static void ad_include(ad_doc_t *d, int i) {
const ad_line_t *l = &d->L[i];
const char *s = l->s;
int n = ad_rlen(l);
static const char kw[] = "include::";
int kl = (int)strlen(kw);
if (n < kl + 2 || memcmp(s, kw, (size_t)kl) != 0 || s[kl] == '[' || ad_blank(s[kl]) ||
s[n - 1] != ']') {
return;
}
const char *open = memchr(s + kl, '[', (size_t)(n - kl));
if (!open) {
return;
}
int ta = kl;
int tb = (int)(open - s);
int aa = tb + 1;
int ab = n - 1;
char *target = ad_expand(d, s + ta, tb - ta);
if (!target || strchr(target, '\t')) {
return;
}
char tags[AD_FIELD_MAX];
char tag1[AD_FIELD_MAX];
char lines[AD_FIELD_MAX];
ad_attrlist_value(s + aa, ab - aa, "tags", tags, sizeof(tags));
ad_attrlist_value(s + aa, ab - aa, "tag", tag1, sizeof(tag1));
ad_attrlist_value(s + aa, ab - aa, "lines", lines, sizeof(lines));
char *written = cbm_arena_strndup(d->ctx->arena, s, (size_t)n);
ad_push(d, i, CBM_DOCLINK_ADOC_INCLUDE,
written ? cbm_arena_sprintf(d->ctx->arena, "%s\t%s\t%s\t%s", written, target,
tags[0] ? tags : tag1, lines)
: NULL);
}
/* Built-in attributes: characters and document settings, never a link. */
static bool ad_builtin(const char *s, size_t n) {
static const char *const names[] = {"nbsp", "sp",
"empty", "blank",
"zwsp", "wj",
"apos", "quot",
"lsquo", "rsquo",
"ldquo", "rdquo",
"deg", "plus",
"brvbar", "vbar",
"amp", "lt",
"gt", "startsb",
"endsb", "caret",
"asterisk", "tilde",
"backslash", "backtick",
"two-colons", "two-semicolons",
"cpp", "cxx",
"pp", "docname",
"docdir", "docfile",
"doctitle", "toc",
"version", "revnumber",
"revdate", "author",
"email", "page-component-version",
NULL};
for (int i = 0; names[i]; i++) {
if (strlen(names[i]) == n && memcmp(names[i], s, n) == 0) {
return true;
}
}
return false;
}
static void ad_attr_refs(ad_doc_t *d, int i) {
const ad_line_t *l = &d->L[i];
const char *s = l->s;
int n = l->len;
for (int k = 0; k < n; k++) {
int a;
int b;
if (!ad_attr_ref(s, n, k, &a, &b) || ad_builtin(s + a, (size_t)(b - a))) {
continue;
}
const ad_attr_t *at = ad_attr_find(d, s + a, (size_t)(b - a));
char *value = at ? ad_expand(d, at->value, (int)strlen(at->value)) : NULL;
if (at && (!value || strchr(value, '\t') || strchr(at->value, '\t'))) {
k = b;
continue;
}
ad_push(d, i, CBM_DOCLINK_ADOC_ATTRIBUTE,
cbm_arena_sprintf(d->ctx->arena, "{%.*s}\t%.*s\t%s\t%s", b - a, s + a, b - a, s + a,
at ? at->value : "", value ? value : ""));
k = b;
}
}
/* Monospace spans of a prose line: `x` and ``x``, a `+x+` passthrough's inner
* text; each a code_path / code_name when the classifier names one. */
static void ad_spans(ad_doc_t *d, int i) {
const ad_line_t *l = &d->L[i];
const char *s = l->s;
int n = l->len;
int k = 0;
while (k < n) {
if (s[k] != '`') {
k++;
continue;
}
int run = 1;
while (k + run < n && s[k + run] == '`') {
run++;
}
if (run > 2) {
k += run;
continue;
}
int a = k + run;
int e = a;
while (e + run <= n && !(memcmp(s + e, run == 2 ? "``" : "`", (size_t)run) == 0)) {
e++;
}
if (e + run > n || e == a) {
k = a;
continue;
}
int b = e;
if (b - a >= 2 && s[a] == '+' && s[b - 1] == '+') {
a++;
b--;
}
CBMDocLinkMdPath p;
if (b > a && b - a <= AD_SPAN_MAX &&
cbm_doclink_md_classify_span(s + a, (size_t)(b - a), d->span_buf, AD_SPAN_MAX + 1,
&p)) {
ad_push(d, i,
p.shape == CBM_DOCLINK_MD_QUALIFIED ? CBM_DOCLINK_ADOC_CODE_NAME
: CBM_DOCLINK_ADOC_CODE_PATH,
cbm_arena_strndup(d->ctx->arena, s + a, (size_t)(b - a)));
}
k = e + run;
}
}
static void ad_tokens(ad_doc_t *d) {
for (int i = 0; i < d->n && !d->failed; i++) {
if (d->in_block[i] == 2) {
continue; /* a comment block: nothing in it is read */
}
const ad_line_t *l = &d->L[i];
if (l->len >= 2 && l->s[0] == '/' && l->s[1] == '/' && !d->in_block[i]) {
continue; /* a line comment */
}
if (!d->in_block[i] && ad_attr_entry(d, l)) {
continue;
}
if (l->len > 0 && l->s[0] == 'i') {
ad_include(d, i);
}
ad_attr_refs(d, i);
if (!d->in_block[i]) {
ad_spans(d, i);
}
}
}
void cbm_adoc_extract_document(CBMExtractCtx *ctx) {
if (!ctx || !ctx->result || !ctx->source || ctx->source_len <= 0) {
return;
}
ad_doc_t d = {.ctx = ctx, .scratch = ctx->scratch ? ctx->scratch : ctx->arena};
d.span_buf = (char *)cbm_arena_alloc(d.scratch, (size_t)(AD_SPAN_MAX + 1) * 2);
if (!d.span_buf || !ad_lines(&d, ctx->source, ctx->source_len)) {
ctx->result->doc_links.failed = true;
return;
}
ad_structure(&d);
if (!d.failed) {
ad_sections(&d);
}
if (!d.failed) {
ad_tokens(&d);
}
if (d.failed) {
ctx->result->doc_links.failed = true;
}
}
/* ── The Antora scope (antora.yml, antora-playbook*.yml) ─────────── */
typedef struct {
CBMArena *a;
char *buf;
size_t len;
size_t cap;
bool failed;
} ad_sb_t;
static void ad_put(ad_sb_t *sb, const char *s, size_t n) {
if (sb->failed) {
return;
}
if (sb->len + n + 1 > sb->cap) {
size_t ncap = sb->cap ? sb->cap : AD_FIELD_MAX;
while (ncap < sb->len + n + 1) {
ncap *= 2;
}
char *grown = (char *)cbm_arena_alloc(sb->a, ncap);
if (!grown) {
sb->failed = true;
return;
}
if (sb->len) {
memcpy(grown, sb->buf, sb->len);
}
sb->buf = grown;
sb->cap = ncap;
}
memcpy(sb->buf + sb->len, s, n);
sb->len += n;
sb->buf[sb->len] = '\0';
}
static void ad_rec(ad_sb_t *sb, const char *kind, const char *a, size_t an, const char *b,
size_t bn) {
for (size_t i = 0; i < an; i++) {
if (a[i] == '\t' || a[i] == '\n') {
return;
}
}
for (size_t i = 0; b && i < bn; i++) {
if (b[i] == '\t' || b[i] == '\n') {
return;
}
}
ad_put(sb, kind, strlen(kind));
ad_put(sb, "\t", 1);
ad_put(sb, a, an);
if (b) {
ad_put(sb, "\t", 1);
ad_put(sb, b, bn);
}
ad_put(sb, "\n", 1);
}
static int ad_indent(const char *s, int n) {
int k = 0;
while (k < n && s[k] == ' ') {
k++;
}
return k;
}
/* `key: value` at s (after the indent): the key and value spans (value
* unquoted, comments not stripped: the field test's reading). */
static bool ad_yaml_kv(const char *s, int n, int *ka, int *kb, int *va, int *vb) {
int k = 0;
if (k >= n || !(ad_word(s[k]))) {
return false;
}
while (k < n && (ad_word(s[k]) || s[k] == '-')) {
k++;
}
int e = k;
while (k < n && ad_blank(s[k])) {
k++;
}
if (k >= n || s[k] != ':') {
return false;
}
k++;
while (k < n && ad_blank(s[k])) {
k++;
}
*ka = 0;
*kb = e;
*va = k;
*vb = n; /* the caller's line has no trailing whitespace */
return true;
}
/* A value in matching quotes loses them. */
static void ad_unquote(const char *s, int *va, int *vb) {
if (*vb - *va >= 2 && s[*va] == s[*vb - 1] && (s[*va] == '\'' || s[*va] == '"')) {
(*va)++;
(*vb)--;
}
}
/* H8 parse_antora_yml: `name:`, `asciidoc: attributes:` and
* `ext: collector: scan: - dir: / into:`. */
static void ad_antora(const char *src, int len, bool component, ad_sb_t *sb) {
bool in_asciidoc = false;
bool in_attrs = false;
int attrs_ind = -1;
bool in_scan = false;
int pos = 0;
while (pos <= len) {
const char *nl = memchr(src + pos, '\n', (size_t)(len - pos));
int end = nl ? (int)(nl - src) : len;
const char *s = src + pos;
int n = end - pos;
while (n > 0 && ad_blank(s[n - 1])) {
n--;
}
int ind = ad_indent(s, n);
int ka;
int kb;
int va;
int vb;
bool kv = ad_yaml_kv(s + ind, n - ind, &ka, &kb, &va, &vb);
if (kv) {
ad_unquote(s + ind, &va, &vb);
}
if (ind == 0 && n > 0) {
in_asciidoc = kv && kb == (int)strlen("asciidoc") && memcmp(s, "asciidoc", 8) == 0;
in_attrs = false;
in_scan = false;
if (component && kv && kb == (int)strlen("name") && memcmp(s, "name", 4) == 0 &&
vb > va) {
ad_rec(sb, "N", s + va, (size_t)(vb - va), NULL, 0);
}
} else if (in_asciidoc && kv && va == vb && kb == (int)strlen("attributes") &&
memcmp(s + ind, "attributes", 10) == 0) {
in_attrs = true;
attrs_ind = -1;
} else if (in_attrs && n > ind && s[ind] != '#') {
if (attrs_ind < 0) {
attrs_ind = ind;
}
if (ind < attrs_ind) {
in_attrs = false;
} else if (ind == attrs_ind && kv) {
ad_rec(sb, "A", s + ind, (size_t)kb, s + ind + va, (size_t)(vb - va));
}
}
if (component && kv && kb == (int)strlen("scan") && memcmp(s + ind, "scan", 4) == 0) {
in_scan = true;
} else if (component && in_scan) {
const char *t = s + ind;
int tn = n - ind;
if (tn > 1 && t[0] == '-' && ad_blank(t[1])) {
int sk = 1;
while (sk < tn && ad_blank(t[sk])) {
sk++;
}
t += sk;
tn -= sk;
}
int a;
int b;
int c;
int e;
bool tkv = ad_yaml_kv(t, tn, &a, &b, &c, &e);
if (tkv) {
ad_unquote(t, &c, &e);
}
if (tkv && e > c) {
if (b == 3 && memcmp(t, "dir", 3) == 0) {
ad_rec(sb, "D", t + c, (size_t)(e - c), NULL, 0);
} else if (b == 4 && memcmp(t, "into", 4) == 0) {
ad_rec(sb, "I", t + c, (size_t)(e - c), NULL, 0);
}
}
}
pos = end + 1;
}
}
const char *cbm_doclink_antora_scan_scope(CBMExtractCtx *ctx) {
if (!ctx || !ctx->rel_path || !ctx->source || ctx->source_len <= 0) {
return NULL;
}
const char *base = strrchr(ctx->rel_path, '/');
base = base ? base + 1 : ctx->rel_path;
bool component = strcmp(base, "antora.yml") == 0;
bool playbook = strncmp(base, "antora-playbook", strlen("antora-playbook")) == 0;
if (!component && !playbook) {
return NULL;
}
ad_sb_t sb = {.a = ctx->arena};
ad_put(&sb, CBM_DOCLINK_ADOC_SCOPE_TAG "\n", strlen(CBM_DOCLINK_ADOC_SCOPE_TAG "\n"));
ad_put(&sb, component ? "C\n" : "P\n", 2);
ad_antora(ctx->source, ctx->source_len, component, &sb);
if (sb.failed) {
ctx->result->doc_links.failed = true;
return NULL;
}
return sb.buf;
}
internal/cbm/doc_adr.c
+1535
View File
File diff suppressed because it is too large Load Diff
internal/cbm/doc_pdf.c
+1392
View File
File diff suppressed because it is too large Load Diff
internal/cbm/doclink.c
+32 -2
View File
@@ -69,7 +69,8 @@ void cbm_doclink_test_note_doc_work(uint64_t copied, uint64_t parse_input, uint6
* doclink.h (language, name, external, and the ship gate). */
static const CBMDocLinkFamily DOCLINK_FAMILIES[CBM_DOCLINK_SYNTAX_COUNT] = {
#define DOCLINK_FAMILY_ROW(id, lang, name, external, ships) [id] = {lang, name, external, ships},
#define DOCLINK_FAMILY_ROW(id, lang, name, external, ships, edge) \
[id] = {lang, name, external, ships, edge},
CBM_DOCLINK_FAMILY_LIST(DOCLINK_FAMILY_ROW)
#undef DOCLINK_FAMILY_ROW
};
@@ -87,6 +88,11 @@ const char *cbm_doclink_syntax_name(int syntax) {
return f ? f->name : "";
}
const char *cbm_doclink_syntax_edge(int syntax) {
const CBMDocLinkFamily *f = cbm_doclink_family(syntax);
return f && f->edge ? f->edge : "MENTIONS";
}
bool cbm_doclink_syntax_is_external(int syntax) {
const CBMDocLinkFamily *f = cbm_doclink_family(syntax);
return f && f->external;
@@ -143,6 +149,9 @@ typedef struct {
* the same doc): only the first label carries the doc's references. */
const char *twin_label;
const char *twin_of;
/* A document language: the whole file is the documentation, so one scan
* of its bytes replaces parse_doc, the file-level doc and the scope. */
void (*scan_file)(CBMExtractCtx *ctx);
} doclink_lang_t;
static const doclink_lang_t DOCLINK_LANGS[] = {
@@ -159,6 +168,18 @@ static const doclink_lang_t DOCLINK_LANGS[] = {
.scan_scope = cbm_doclink_cs_project_scan_scope,
.scope_tag = CBM_DOCLINK_CS_SCOPE_TAG,
.portable_scope = cbm_doclink_cs_portable_scope},
{.lang = CBM_LANG_MARKDOWN, .scan_file = cbm_doclink_md_scan_file},
{.lang = CBM_LANG_RST, .scan_file = cbm_doclink_rst_scan_file},
/* Python: no doc references yet, only the scope the reST resolver reads
* (package re-exports, Sphinx conf.py settings) */
{.lang = CBM_LANG_PYTHON,
.scan_scope = cbm_doclink_py_scan_scope,
.scope_tag = CBM_DOCLINK_PY_SCOPE_TAG},
/* YAML: the Antora component and playbook settings the AsciiDoc resolver
* reads */
{.lang = CBM_LANG_YAML,
.scan_scope = cbm_doclink_antora_scan_scope,
.scope_tag = CBM_DOCLINK_ADOC_SCOPE_TAG},
};
static const doclink_lang_t *doclink_lang(CBMLanguage lang) {
@@ -171,7 +192,8 @@ static const doclink_lang_t *doclink_lang(CBMLanguage lang) {
}
bool cbm_doclink_lang_supported(CBMLanguage lang) {
return doclink_lang(lang) != NULL;
const doclink_lang_t *L = doclink_lang(lang);
return L && (L->parse_doc || L->scan_file);
}
/* ── Doc-line side map ───────────────────────────────────────────── */
@@ -388,6 +410,14 @@ void cbm_doclinks_extract(CBMExtractCtx *ctx) {
if (!L) {
return;
}
if (L->scan_file) {
L->scan_file(ctx);
return;
}
if (!L->parse_doc) { /* a scope, and no references (Python) */
ctx->result->doc_scope = L->scan_scope ? L->scan_scope(ctx) : NULL;
return;
}
int twin_count = 0;
doclink_twin_key_t *twins = L->twin_label ? twin_keys(ctx, L->twin_of, &twin_count) : NULL;
CBMDefArray *defs = &ctx->result->defs;
internal/cbm/doclink.h
+178 -13
View File
@@ -22,6 +22,10 @@
* identical on full and incremental runs. Optional.
* scope_tag, the blob's tag line and its persisted form (line
* portable_scope numbers dropped). Optional.
* scan_file instead of parse_doc, for a DOCUMENT language whose
* text is the documentation (Markdown): one pass over
* the file's bytes pushes every reference, with the
* section it is written in as its source. Optional.
* State a language's hooks need between their calls for one file lives in
* ctx->doclink_state (cbm.h), never in a static or thread-local.
* A language without a row produces no tokens and no scope. The resolving
@@ -39,9 +43,9 @@
/* Every link family, across languages: one line per (language, reference
* form), and the ONE place a language leg adds its families --
*
* X(enum value, language, name, external, ships)
* X(enum value, language, name, external, ships, edge)
*
* name the "syntax" MENTIONS-edge property and doc_link_unresolved
* name the "syntax" edge property and doc_link_unresolved
* column, so it is public surface; with the language it is the
* tier a release audit judges. Names repeat across languages
* (`see` is C#'s, Java's, Kotlin's ...): the value, not the name,
@@ -51,22 +55,73 @@
* edge. false: the tier is below the audit's bar; its references
* still resolve, but a resolved one stays a doc_link_unresolved
* row with reason below_bar_tier (an unresolved one keeps its own
* reason). Nothing else about the family changes.
* reason). Nothing else about the family changes. A document
* family ships once a held-out audit passed it: below the bar
* (AsciiDoc code spans naming paths, AsciiDoc code names, reST
* object directives; Markdown code spans naming a file or one
* directory alone, split off the Markdown code-path tier after
* its held-out failure; ADR supersedes words in running prose,
* split off after three held-out censuses failed on them), or
* not yet audited for want of held-out occurrences (reST
* include, kernel_doc; AsciiDoc attribute) means false.
* Markdown code paths without the bare part, reST
* literalinclude and code paths, PDF qualified names after the
* owner rule, and ADR supersedes statements (the phrase opens
* its line or field and names a record: 80 of 81 on a fresh
* census) passed supplementary held-out audits.
* edge the type of the edge a resolved reference becomes: MENTIONS
* (documentation names code), SUPERSEDES (an ADR states that it
* supersedes another). The edge's source is always the
* definition the reference is written in, so a file owns the
* edges its text produces, whatever their type.
*
* The enum below and the family table (doclink.c) are both generated from this
* list, so a value cannot exist without its name and its gate. */
#define CBM_DOCLINK_FAMILY_LIST(X) \
/* csharp */ \
X(CBM_DOCLINK_CS_SEE, CBM_LANG_CSHARP, "see", false, true) \
X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true) \
X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true) \
X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true) \
/* any language (CBM_LANG_COUNT): a URL is never an edge */ \
X(CBM_DOCLINK_HREF, CBM_LANG_COUNT, "href", true, false)
#define CBM_DOCLINK_FAMILY_LIST(X) \
/* csharp */ \
X(CBM_DOCLINK_CS_SEE, CBM_LANG_CSHARP, "see", false, true, "MENTIONS") \
X(CBM_DOCLINK_CS_SEEALSO, CBM_LANG_CSHARP, "seealso", false, true, "MENTIONS") \
X(CBM_DOCLINK_CS_EXCEPTION, CBM_LANG_CSHARP, "exception", false, true, "MENTIONS") \
X(CBM_DOCLINK_CS_INHERITDOC, CBM_LANG_CSHARP, "inheritdoc", false, true, "MENTIONS") \
/* markdown (doclink_md.c): what a document's text names explicitly */ \
X(CBM_DOCLINK_MD_LINK, CBM_LANG_MARKDOWN, "link", false, true, "MENTIONS") \
X(CBM_DOCLINK_MD_PATH, CBM_LANG_MARKDOWN, "path", false, true, "MENTIONS") \
X(CBM_DOCLINK_MD_CODE_PATH, CBM_LANG_MARKDOWN, "code_path", false, true, "MENTIONS") \
/* a code span naming a file or directory alone (`config.yaml`, `doc/`) */ \
X(CBM_DOCLINK_MD_BARE_PATH, CBM_LANG_MARKDOWN, "bare_path", false, false, "MENTIONS") \
X(CBM_DOCLINK_MD_CODE_NAME, CBM_LANG_MARKDOWN, "code_name", false, true, "MENTIONS") \
/* an ADR's own "supersedes" / "replaces" statement, opening its line or field */ \
/* and naming a record directly (doc_adr.c): ADR -> ADR */ \
X(CBM_DOCLINK_MD_SUPERSEDES, CBM_LANG_MARKDOWN, "supersedes", false, true, "SUPERSEDES") \
/* the same words in running prose ("this record supersedes ADR-3") */ \
X(CBM_DOCLINK_MD_SUPERSEDES_PROSE, CBM_LANG_MARKDOWN, "supersedes_prose", false, false, \
"SUPERSEDES") \
/* pdf (doc_pdf.c): structural mentions in a page's text; pdf_name (bare */ \
/* names) only decides which line fragments a cross-line join replaces */ \
X(CBM_DOCLINK_PDF_PATH, CBM_LANG_PDF, "pdf_path", false, true, "MENTIONS") \
X(CBM_DOCLINK_PDF_FILE, CBM_LANG_PDF, "pdf_file", false, true, "MENTIONS") \
X(CBM_DOCLINK_PDF_QN, CBM_LANG_PDF, "pdf_qn", false, true, "MENTIONS") \
X(CBM_DOCLINK_PDF_NAME, CBM_LANG_PDF, "pdf_name", false, false, "MENTIONS") \
/* restructuredtext (doclink_rst.c): Sphinx domain markup, directives, inline literals */ \
X(CBM_DOCLINK_RST_ROLE, CBM_LANG_RST, "role", false, true, "MENTIONS") \
X(CBM_DOCLINK_RST_OBJECT, CBM_LANG_RST, "object", false, true, "MENTIONS") \
X(CBM_DOCLINK_RST_AUTODOC, CBM_LANG_RST, "autodoc", false, true, "MENTIONS") \
X(CBM_DOCLINK_RST_INCLUDE, CBM_LANG_RST, "include", false, false, "MENTIONS") \
X(CBM_DOCLINK_RST_LITERALINCLUDE, CBM_LANG_RST, "literalinclude", false, true, "MENTIONS") \
X(CBM_DOCLINK_RST_KERNEL_DOC, CBM_LANG_RST, "kernel_doc", false, false, "MENTIONS") \
X(CBM_DOCLINK_RST_CODE_PATH, CBM_LANG_RST, "code_path", false, true, "MENTIONS") \
X(CBM_DOCLINK_RST_CODE_NAME, CBM_LANG_RST, "code_name", false, true, "MENTIONS") \
/* asciidoc (doc_adoc.c): includes, Antora API attributes, monospace spans */ \
X(CBM_DOCLINK_ADOC_INCLUDE, CBM_LANG_ASCIIDOC, "include", false, true, "MENTIONS") \
X(CBM_DOCLINK_ADOC_ATTRIBUTE, CBM_LANG_ASCIIDOC, "attribute", false, false, "MENTIONS") \
X(CBM_DOCLINK_ADOC_CODE_PATH, CBM_LANG_ASCIIDOC, "code_path", false, false, "MENTIONS") \
X(CBM_DOCLINK_ADOC_CODE_NAME, CBM_LANG_ASCIIDOC, "code_name", false, false, "MENTIONS") \
/* any language (CBM_LANG_COUNT): a URL is never an edge */ \
X(CBM_DOCLINK_HREF, CBM_LANG_COUNT, "href", true, false, "MENTIONS")
typedef enum {
CBM_DOCLINK_NONE = 0,
#define CBM_DOCLINK_FAMILY_VALUE(id, lang, name, external, ships) id,
#define CBM_DOCLINK_FAMILY_VALUE(id, lang, name, external, ships, edge) id,
CBM_DOCLINK_FAMILY_LIST(CBM_DOCLINK_FAMILY_VALUE)
#undef CBM_DOCLINK_FAMILY_VALUE
CBM_DOCLINK_SYNTAX_COUNT
@@ -79,8 +134,13 @@ enum {
* edge source is the file's File node, which the resolving half looks up
* from the file it is resolving; `source_qn` is not the source then. Set
* by the driver (cbm_doclinks_extract) on what a parser pushes for the
* file-level doc -- a parser never sets it. */
* file-level doc -- a parser never sets it. A scan_file hook sets it on a
* reference written before the document's first section. */
CBM_DOCLINK_FLAG_FILE = 1,
/* A PDF mention that runs across a line break, proposed as one reference.
* `raw` holds its forms (separated by 0x1F) and, after 0x1E, the indices
* of the line-local tokens it replaces when it resolves (doc_pdf.c). */
CBM_DOCLINK_FLAG_JOIN = 2,
};
/* One link family: a line of CBM_DOCLINK_FAMILY_LIST. */
@@ -89,8 +149,12 @@ typedef struct {
const char *name;
bool external;
bool ships;
const char *edge; /* the type of the edge a resolved reference becomes */
} CBMDocLinkFamily;
/* "MENTIONS" or "SUPERSEDES"; "MENTIONS" for an unknown value. */
const char *cbm_doclink_syntax_edge(int syntax);
/* The family of a syntax value; NULL for a value that names none. */
const CBMDocLinkFamily *cbm_doclink_family(int syntax);
@@ -221,4 +285,105 @@ size_t cbm_doclink_cs_norm_type(const char *in, size_t len, char *out, size_t ca
/* The C# scope blob starts with this tag line. */
#define CBM_DOCLINK_CS_SCOPE_TAG "cs1"
/* ── Markdown (doclink_md.c) ───────────────────────────────────────── */
/* The scan_file hook: every explicit reference of a Markdown or MDX file (a
* link destination, a bare path, a path or a qualified name in a code span),
* each with the Section definition it is written under as its source. */
void cbm_doclink_md_scan_file(CBMExtractCtx *ctx);
/* What a code span's text names. */
typedef enum {
CBM_DOCLINK_MD_NONE = 0, /* nothing linkable (a bare name, a command, a snippet, ...) */
CBM_DOCLINK_MD_FILE, /* a path with a directory part, or a dot file */
CBM_DOCLINK_MD_FILENAME, /* a file name alone: it names a file of the document's
* own directory, never one found elsewhere */
CBM_DOCLINK_MD_DIR, /* a directory */
CBM_DOCLINK_MD_QUALIFIED, /* a qualified name of two or more identifiers
* (`a.b`, `a::b`, `A#b`, `A\B`, `mod:attr`) */
} CBMDocLinkMdShape;
typedef struct {
CBMDocLinkMdShape shape;
const char *path; /* NUL-terminated, in the caller's buffer; for QUALIFIED the
* identifiers joined by '.' */
const char *member; /* the text after a `path::`, NUL-terminated in the same
* buffer: the member the path's file declares; NULL: none */
uint32_t first_line; /* a line range written with the path (`#L3-L9`, `:3-9`, `:3`); */
uint32_t last_line; /* 0 and 0 when there is none */
bool instance; /* QUALIFIED: written through an instance (`self.x`, `this.x`):
* the qualifier is no type, so it never names a field */
bool colon; /* QUALIFIED: written `module:attr` (Python's entry-point form) */
} CBMDocLinkMdPath;
/* Classify the text of a code span. Returns false, with shape NONE, when it
* names no path and no qualified name, or when `buf` (cap bytes) cannot hold
* it: a name is never cut. Pure: the same text always gives the same answer. */
bool cbm_doclink_md_classify_span(const char *text, size_t len, char *buf, size_t cap,
CBMDocLinkMdPath *out);
/* Parse a `#L3`, `#L3-L9` or `#L3C2-L9C5` fragment (without the `#`): true
* and the lines, or false. */
bool cbm_doclink_md_line_fragment(const char *frag, uint32_t *first, uint32_t *last);
/* ── ADRs (doc_adr.c) ──────────────────────────────────────────────── */
/* When the Markdown file being extracted is an architecture decision record:
* push its ADR definition (name its canonical id, qualified name
* "<module>.__adr__", facts as extra properties) and a `supersedes` token for
* each of its own supersedes / replaces statements. Called by the Markdown
* scan, after the Section definitions exist. */
void cbm_adr_extract(CBMExtractCtx *ctx);
/* The qualified-name tail of an ADR node: "<module QN>" + this. */
#define CBM_ADR_QN_NAME "__adr__"
/* ── reStructuredText (doclink_rst.c) and Python's scope (doclink_py.c) ── */
/* The scan_file hook of a reStructuredText document: a Section per title
* (the text up to the next title as its docstring) and its references to
* code -- Sphinx roles and object directives, autodoc, include /
* literalinclude / kernel-doc paths, inline literals naming a path or a
* qualified name -- each with the section it is written in as its source. */
void cbm_doclink_rst_scan_file(CBMExtractCtx *ctx);
/* The scope blob of a Python file, which the reST resolver reads: a package
* `__init__.py`'s top-level explicit `from x import y [as z]` (what the
* package re-exports), and a Sphinx `conf.py`'s primary domain, extlink roles
* to repository paths and intersphinx names (read as text, never run). NULL
* for every other file. */
const char *cbm_doclink_py_scan_scope(CBMExtractCtx *ctx);
/* The Python scope blob starts with this tag line. */
#define CBM_DOCLINK_PY_SCOPE_TAG "py1"
/* ── AsciiDoc (doc_adoc.c) ─────────────────────────────────────────── */
/* AsciiDoc documents: a Section per heading and the references to code
* (include::, Antora API attributes, monospace spans) as doc-link tokens.
* Runs instead of a grammar for CBM_LANG_ASCIIDOC. */
void cbm_adoc_extract_document(CBMExtractCtx *ctx);
/* The scope blob of a YAML file, which the AsciiDoc resolver reads: an
* antora.yml's component name, asciidoc attributes and collector scans, an
* antora-playbook*.yml's attributes (read as text). NULL for every other
* file. */
const char *cbm_doclink_antora_scan_scope(CBMExtractCtx *ctx);
/* The Antora scope blob starts with this tag line. */
#define CBM_DOCLINK_ADOC_SCOPE_TAG "ad1"
/* PDF documents (doc_pdf.c): a Section per page (name "page N", the page's
* text as docstring, property page) and the pages' structural code mentions
* as doc-link tokens. Runs instead of a grammar for CBM_LANG_PDF. */
void cbm_pdf_extract_document(CBMExtractCtx *ctx);
#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
/* Test seam: scan one page's text as cbm_pdf_extract_document does (tokens
* into ctx->result->doc_links), without a PDF around it. */
void cbm_pdf_test_scan_page(CBMExtractCtx *ctx, const char *text, size_t len, uint32_t page,
const char *page_qn);
/* candidates the scanner's joins have compared so far (a cost counter) */
size_t cbm_pdf_test_join_steps(void);
#endif
#endif /* CBM_DOCLINK_H */
internal/cbm/doclink_md.c
+1812
View File
File diff suppressed because it is too large Load Diff
internal/cbm/doclink_py.c
+419
View File
@@ -0,0 +1,419 @@
/*
* doclink_py.c — the doc-link scope of a Python file: what the reST resolver
* (src/pipeline/doc_links_rst.c) needs from Python sources to resolve Sphinx
* references the way Sphinx does.
*
* I \t level \t module \t name \t alias a package `__init__.py`'s top-level
* `from module import name [as alias]`:
* the package re-exports `alias`
* (`django.db.models.Model` is
* django/db/models/base.py's Model).
* Star imports are not read (the field
* test's resolver did not follow them).
* C the file is a `conf.py`: documents in
* and below its directory are one
* Sphinx documentation set
* D \t domain its primary_domain
* X \t role an extlink role whose URL is a
* repository blob/tree path
* (:source:`django/db/x.py`)
* S \t name an intersphinx name (`python:` in
* :class:`python:dict` is external)
*
* conf.py is read as text, with the field test's patterns, and never run.
* Every other Python file has no scope (NULL), so a change to it never
* changes another file's resolution through this blob.
*/
#include "doclink.h"
#include "arena.h"
#include "cbm.h"
#include <stdbool.h>
#include <stddef.h>
#include <stdio.h>
#include <string.h>
enum {
PY_SCOPE_FIELD_MAX = 256, /* a module, name or setting longer than this is skipped */
PY_SB_INIT = 256,
PY_INT_DIGITS = 16,
};
/* A small arena string builder (the blob lives in the result arena). */
typedef struct {
CBMArena *a;
char *buf;
size_t len;
size_t cap;
bool failed;
} py_sb_t;
static void py_put(py_sb_t *sb, const char *s, size_t n) {
if (sb->failed) {
return;
}
if (sb->len + n + 1 > sb->cap) {
size_t ncap = sb->cap ? sb->cap : PY_SB_INIT;
while (ncap < sb->len + n + 1) {
ncap *= 2;
}
char *grown = (char *)cbm_arena_alloc(sb->a, ncap);
if (!grown) {
sb->failed = true;
return;
}
if (sb->len > 0) {
memcpy(grown, sb->buf, sb->len);
}
sb->buf = grown;
sb->cap = ncap;
}
memcpy(sb->buf + sb->len, s, n);
sb->len += n;
sb->buf[sb->len] = '\0';
}
/* One record: its kind, then each field after a TAB, then a newline. */
static void py_rec(py_sb_t *sb, const char *kind, int nfields, const char *const *f,
const size_t *fl) {
py_put(sb, kind, strlen(kind));
for (int i = 0; i < nfields; i++) {
py_put(sb, "\t", 1);
py_put(sb, f[i], fl[i]);
}
py_put(sb, "\n", 1);
}
static const char *py_base(const char *path) {
const char *slash = strrchr(path, '/');
return slash ? slash + 1 : path;
}
static bool py_field_ok(const char *s, size_t n) {
if (n == 0 || n > PY_SCOPE_FIELD_MAX) {
return false;
}
for (size_t i = 0; i < n; i++) {
unsigned char c = (unsigned char)s[i];
if (c == '\t' || c == '\n' || c == '\r' || c < ' ') {
return false;
}
}
return true;
}
static void py_text(const CBMExtractCtx *ctx, TSNode n, const char **s, size_t *len) {
uint32_t a = ts_node_start_byte(n);
uint32_t b = ts_node_end_byte(n);
if (b > (uint32_t)ctx->source_len || a > b) {
*s = "";
*len = 0;
return;
}
*s = ctx->source + a;
*len = b - a;
}
/* One `from module import names` statement of the package's top level. */
static void py_from_import(const CBMExtractCtx *ctx, TSNode stmt, py_sb_t *sb) {
TSNode mod = ts_node_child_by_field_name(stmt, "module_name", (uint32_t)strlen("module_name"));
if (ts_node_is_null(mod)) {
return;
}
int level = 0;
const char *ms = "";
size_t ml = 0;
if (strcmp(ts_node_type(mod), "relative_import") == 0) {
uint32_t nc = ts_node_child_count(mod);
for (uint32_t i = 0; i < nc; i++) {
TSNode c = ts_node_child(mod, i);
const char *k = ts_node_type(c);
if (strcmp(k, "import_prefix") == 0) {
const char *ps;
size_t pl;
py_text(ctx, c, &ps, &pl);
for (size_t j = 0; j < pl; j++) {
level += ps[j] == '.';
}
} else if (strcmp(k, "dotted_name") == 0) {
py_text(ctx, c, &ms, &ml);
}
}
} else {
py_text(ctx, mod, &ms, &ml);
}
if (ml > 0 && !py_field_ok(ms, ml)) {
return;
}
uint32_t nc = ts_node_child_count(stmt);
for (uint32_t i = 0; i < nc; i++) {
const char *field = ts_node_field_name_for_child(stmt, i);
if (!field || strcmp(field, "name") != 0) {
continue;
}
TSNode c = ts_node_child(stmt, i);
TSNode name = c;
TSNode alias = c;
if (strcmp(ts_node_type(c), "aliased_import") == 0) {
name = ts_node_child_by_field_name(c, "name", (uint32_t)strlen("name"));
alias = ts_node_child_by_field_name(c, "alias", (uint32_t)strlen("alias"));
if (ts_node_is_null(name) || ts_node_is_null(alias)) {
continue;
}
}
const char *ns;
size_t nl;
const char *as;
size_t al;
py_text(ctx, name, &ns, &nl);
py_text(ctx, alias, &as, &al);
if (!py_field_ok(ns, nl) || !py_field_ok(as, al)) {
continue;
}
char lv[PY_INT_DIGITS];
int lvn = snprintf(lv, sizeof(lv), "%d", level);
const char *f[] = {lv, ms, ns, as};
const size_t fl[] = {lvn > 0 ? (size_t)lvn : 0, ml, nl, al};
py_rec(sb, "I", 4, f, fl);
}
}
/* ── conf.py, read as text (the field test's patterns) ───────────── */
static bool py_ws(char c) {
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v';
}
static bool py_word(char c) {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
}
static size_t py_skip_ws(const char *s, size_t n, size_t i) {
while (i < n && py_ws(s[i])) {
i++;
}
return i;
}
/* `^name\s*=\s*` at a line start: the index after it, or 0. */
static size_t py_assign_at(const char *s, size_t n, size_t line, const char *name) {
size_t nl = strlen(name);
if (line + nl > n || memcmp(s + line, name, nl) != 0) {
return 0;
}
size_t i = py_skip_ws(s, n, line + nl);
if (i >= n || s[i] != '=') {
return 0;
}
return py_skip_ws(s, n, i + 1);
}
/* conf_scalar: ^name\s*=\s*["']([^"']*)["'] (first match). */
static bool py_conf_scalar(const char *s, size_t n, const char *name, const char **v, size_t *vl) {
for (size_t line = 0; line < n;) {
size_t i = py_assign_at(s, n, line, name);
if (i > 0 && i < n && (s[i] == '"' || s[i] == '\'')) {
size_t j = i + 1;
while (j < n && s[j] != '"' && s[j] != '\'') {
j++;
}
if (j < n) {
*v = s + i + 1;
*vl = j - i - 1;
return true;
}
}
const char *nl = memchr(s + line, '\n', n - line);
line = nl ? (size_t)(nl - s) + 1 : n;
}
return false;
}
/* conf_dict_block: ^name\s*=\s*\{(.*?)^\} -- the text between, or false. */
static bool py_conf_block(const char *s, size_t n, const char *name, size_t *a, size_t *b) {
for (size_t line = 0; line < n;) {
size_t i = py_assign_at(s, n, line, name);
if (i > 0 && i < n && s[i] == '{') {
size_t k = i + 1;
while (k < n) {
if (s[k] == '}' && (k == 0 || s[k - 1] == '\n')) {
*a = i + 1;
*b = k;
return true;
}
k++;
}
return false;
}
const char *nl = memchr(s + line, '\n', n - line);
line = nl ? (size_t)(nl - s) + 1 : n;
}
return false;
}
/* ["']([\w.-]+)["'] at s[i] (extra: also '.'): its end after the closing quote, or 0. */
static size_t py_quoted_key(const char *s, size_t n, size_t i, bool dots, size_t *ka, size_t *kb) {
if (i >= n || (s[i] != '"' && s[i] != '\'')) {
return 0;
}
size_t j = i + 1;
while (j < n && (py_word(s[j]) || s[j] == '-' || (dots && s[j] == '.'))) {
j++;
}
if (j == i + 1 || j >= n || (s[j] != '"' && s[j] != '\'')) {
return 0;
}
*ka = i + 1;
*kb = j;
return j + 1;
}
/* ["']([^"']+)["'] at s[i]: the value span, or false. */
static bool py_quoted_value(const char *s, size_t n, size_t i, size_t *va, size_t *vb) {
if (i >= n || (s[i] != '"' && s[i] != '\'')) {
return false;
}
size_t j = i + 1;
while (j < n && s[j] != '"' && s[j] != '\'') {
j++;
}
if (j == i + 1 || j >= n) {
return false;
}
*va = i + 1;
*vb = j;
return true;
}
/* /(blob|tree)/[^/]+/%s in the URL: a repository path pattern. */
static bool py_repo_url(const char *u, size_t n) {
static const char *const kinds[] = {"/blob/", "/tree/", NULL};
for (int k = 0; kinds[k]; k++) {
size_t kl = strlen(kinds[k]);
for (size_t i = 0; i + kl <= n; i++) {
if (memcmp(u + i, kinds[k], kl) != 0) {
continue;
}
size_t j = i + kl;
size_t ref = j;
while (j < n && u[j] != '/') {
j++;
}
if (j > ref && j + 3 <= n && memcmp(u + j, "/%s", 3) == 0) {
return true;
}
}
}
return false;
}
static void py_extlink(py_sb_t *sb, const char *s, size_t ka, size_t kb, size_t va, size_t vb) {
if (py_repo_url(s + va, vb - va) && py_field_ok(s + ka, kb - ka)) {
const char *f[] = {s + ka};
const size_t fl[] = {kb - ka};
py_rec(sb, "X", 1, f, fl);
}
}
static void py_conf(const char *s, size_t n, py_sb_t *sb) {
py_rec(sb, "C", 0, NULL, NULL);
const char *v;
size_t vl;
if (py_conf_scalar(s, n, "primary_domain", &v, &vl) && py_field_ok(v, vl)) {
const char *f[] = {v};
const size_t fl[] = {vl};
py_rec(sb, "D", 1, f, fl);
}
size_t a;
size_t b;
if (py_conf_block(s, n, "intersphinx_mapping", &a, &b)) {
for (size_t i = a; i < b; i++) {
size_t ka;
size_t kb;
size_t e = py_quoted_key(s, b, i, true, &ka, &kb);
if (e == 0) {
continue;
}
size_t c = py_skip_ws(s, b, e);
if (c < b && s[c] == ':' && (c = py_skip_ws(s, b, c + 1)) < b && s[c] == '(' &&
py_field_ok(s + ka, kb - ka)) {
const char *f[] = {s + ka};
const size_t fl[] = {kb - ka};
py_rec(sb, "S", 1, f, fl);
}
i = e - 1;
}
}
/* extlinks = {'role': ('url%s', ...)} and extlinks['role'] = ('url%s', ...) */
if (py_conf_block(s, n, "extlinks", &a, &b)) {
for (size_t i = a; i < b; i++) {
size_t ka;
size_t kb;
size_t e = py_quoted_key(s, b, i, false, &ka, &kb);
if (e == 0) {
continue;
}
size_t c = py_skip_ws(s, b, e);
size_t va;
size_t vb;
if (c < b && s[c] == ':' && (c = py_skip_ws(s, b, c + 1)) < b && s[c] == '(' &&
py_quoted_value(s, b, py_skip_ws(s, b, c + 1), &va, &vb)) {
py_extlink(sb, s, ka, kb, va, vb);
}
i = e - 1;
}
}
static const char key_open[] = "extlinks[";
size_t kol = strlen(key_open);
for (size_t at = 0; at + kol <= n; at++) {
if (memcmp(s + at, key_open, kol) != 0) {
continue;
}
size_t i = at + kol;
size_t ka;
size_t kb;
size_t e = py_quoted_key(s, n, i, false, &ka, &kb);
if (e == 0 || e >= n || s[e] != ']') {
continue;
}
size_t c = py_skip_ws(s, n, e + 1);
size_t va;
size_t vb;
if (c < n && s[c] == '=' && (c = py_skip_ws(s, n, c + 1)) < n && s[c] == '(' &&
py_quoted_value(s, n, py_skip_ws(s, n, c + 1), &va, &vb)) {
py_extlink(sb, s, ka, kb, va, vb);
}
}
}
const char *cbm_doclink_py_scan_scope(CBMExtractCtx *ctx) {
if (!ctx || !ctx->rel_path || !ctx->source) {
return NULL;
}
const char *base = py_base(ctx->rel_path);
bool init = strcmp(base, "__init__.py") == 0;
bool conf = strcmp(base, "conf.py") == 0;
if (!init && !conf) {
return NULL;
}
py_sb_t sb = {.a = ctx->arena};
py_put(&sb, CBM_DOCLINK_PY_SCOPE_TAG "\n", strlen(CBM_DOCLINK_PY_SCOPE_TAG "\n"));
if (conf) {
py_conf(ctx->source, (size_t)ctx->source_len, &sb);
}
if (init && !ts_node_is_null(ctx->root)) {
uint32_t nc = ts_node_child_count(ctx->root);
for (uint32_t i = 0; i < nc; i++) {
TSNode c = ts_node_child(ctx->root, i);
if (strcmp(ts_node_type(c), "import_from_statement") == 0) {
py_from_import(ctx, c, &sb);
}
}
}
if (sb.failed) {
ctx->result->doc_links.failed = true;
return NULL;
}
return sb.buf;
}
internal/cbm/doclink_rst.c
+1791
View File
File diff suppressed because it is too large Load Diff
internal/cbm/extract_defs.c
+84 -7
View File
@@ -221,7 +221,8 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
static void extract_variables(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec);
static void extract_var_names(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec);
static void extract_class_variables(CBMExtractCtx *ctx, TSNode class_node, const char *class_qn,
const CBMLangSpec *spec);
const CBMLangSpec *spec, int fields_from);
static void fence_fields_named_like_methods(CBMExtractCtx *ctx, int methods_from, int fields_from);
static void extract_rust_impl(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec);
static void extract_class_methods(CBMExtractCtx *ctx, TSNode class_node, const char *class_qn,
const CBMLangSpec *spec);
@@ -7621,13 +7622,16 @@ static void emit_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *s
}
// Extract methods inside the class
int methods_from = ctx->result->defs.count;
extract_class_methods(ctx, node, class_qn, spec);
// Extract typed struct/class fields (for cross-file LSP type resolution)
int fields_from = ctx->result->defs.count;
extract_class_fields(ctx, node, class_qn, spec);
fence_fields_named_like_methods(ctx, methods_from, fields_from);
// Extract class-level variables (field declarations)
extract_class_variables(ctx, node, class_qn, spec);
// Extract class-level variables (field declarations not already Fields)
extract_class_variables(ctx, node, class_qn, spec, fields_from);
if (ctx->language == CBM_LANG_PYTHON) {
extract_py_field_types(ctx, node, class_qn);
@@ -10263,6 +10267,15 @@ static TSNode resolve_field_name_node(TSNode child) {
return null_node;
}
const char *nk = ts_node_type(name_node);
/* Java `private static final String PROP = "v";`: the declarator holds the
* initializer too, so its text named the Field `PROP = "v"`; its name is
* the declarator's own name field */
if (strcmp(nk, "variable_declarator") == 0) {
TSNode inner = ts_node_child_by_field_name(name_node, TS_FIELD("name"));
if (!ts_node_is_null(inner)) {
return inner;
}
}
if (strcmp(nk, "pointer_declarator") == 0 || strcmp(nk, "array_declarator") == 0) {
TSNode inner = ts_node_child_by_field_name(name_node, TS_FIELD("declarator"));
if (!ts_node_is_null(inner)) {
@@ -10661,9 +10674,59 @@ static void extract_class_fields(CBMExtractCtx *ctx, TSNode class_node, const ch
member_iter_done(&it);
}
// Extract class-level variables (field declarations inside class bodies)
static int cmp_cstr_ptr(const void *pa, const void *pb) {
return strcmp(*(const char *const *)pa, *(const char *const *)pb);
}
// A field and a method of one class body may share a name (Java `count` and
// `count()`), and the graph keeps one node per qualified name, so one of them
// was lost. The field then takes the qualified name `<Owner>.<name>#field`
// (the `base#suffix` fence of `#macro` and the Rust cfg twins; the registry
// still files it under its plain name). Only a collision in the same body
// renames, so every other field keeps its name. The body's methods are the
// defs [methods_from, fields_from), its fields [fields_from, count); a sorted
// copy of the method names keeps a large class linear-logarithmic.
static void fence_fields_named_like_methods(CBMExtractCtx *ctx, int methods_from, int fields_from) {
CBMDefinition *defs = ctx->result->defs.items;
int fields_to = ctx->result->defs.count;
if (fields_from >= fields_to || methods_from >= fields_from) {
return;
}
const char **qns = (const char **)cbm_arena_alloc(
ctx->arena, (size_t)(fields_from - methods_from) * sizeof(const char *));
if (!qns) {
return;
}
size_t n = 0;
for (int i = methods_from; i < fields_from; i++) {
if (defs[i].qualified_name && strcmp(defs[i].label, "Method") == 0) {
qns[n++] = defs[i].qualified_name;
}
}
if (n == 0) {
return;
}
qsort(qns, n, sizeof(qns[0]), cmp_cstr_ptr);
for (int i = fields_from; i < fields_to; i++) {
const char *qn = defs[i].qualified_name;
if (qn && strcmp(defs[i].label, "Field") == 0 &&
bsearch(&qn, qns, n, sizeof(qns[0]), cmp_cstr_ptr)) {
char *fenced = cbm_arena_sprintf(ctx->arena, "%s#field", qn);
if (fenced) {
defs[i].qualified_name = fenced;
}
}
}
}
// Extract class-level variables (field declarations inside class bodies).
// A declaration that extract_class_fields already made a Field (the defs from
// `fields_from` on) is not minted again: Java, C# and Solidity list the same
// kind as field and variable, and the second copy was a Variable with the
// module's qualified name (`demo.query.filters` for field Query.filters),
// which collided with package paths.
static void extract_class_variables(CBMExtractCtx *ctx, TSNode class_node, const char *class_qn,
const CBMLangSpec *spec) {
const CBMLangSpec *spec, int fields_from) {
if (!spec->variable_node_types || !spec->variable_node_types[0]) {
return;
}
@@ -10673,6 +10736,10 @@ static void extract_class_variables(CBMExtractCtx *ctx, TSNode class_node, const
return;
}
/* the Fields of this body, in body order: walked alongside the children */
int fields_to = ctx->result->defs.count;
int fi = fields_from;
/* Record the declaring class on every variable minted from this body (see
* push_var_def_qn); saved/restored so module-level minting stays bare. */
const char *saved_parent = ctx->var_parent_class;
@@ -10681,9 +10748,19 @@ static void extract_class_variables(CBMExtractCtx *ctx, TSNode class_node, const
member_iter_init(&it, body, ctx->language, true);
TSNode child;
while (member_iter_next(&it, &child)) {
if (cbm_kind_in_set(child, spec->variable_node_types)) {
extract_var_names(ctx, child, spec);
if (!cbm_kind_in_set(child, spec->variable_node_types)) {
continue;
}
int line = (int)ts_node_start_point(child).row + TS_LINE_OFFSET;
while (fi < fields_to && (strcmp(ctx->result->defs.items[fi].label, "Field") != 0 ||
(int)ctx->result->defs.items[fi].start_line < line)) {
fi++;
}
if (fi < fields_to && (int)ctx->result->defs.items[fi].start_line == line &&
spec->field_node_types && cbm_kind_in_set(child, spec->field_node_types)) {
continue; /* already a Field of this class */
}
extract_var_names(ctx, child, spec);
}
member_iter_done(&it);
ctx->var_parent_class = saved_parent;
internal/cbm/lang_specs.c
+8
View File
@@ -2713,6 +2713,14 @@ static const CBMLangSpec lang_specs[CBM_LANG_COUNT] = {
empty_types, NULL, NULL, tree_sitter_objectscript_routine,
NULL},
// CBM_LANG_PDF — PDF documents. No grammar row: extraction reads the text
// layer itself (cbm_pdf_extract_document, doc_pdf.c) before any grammar is
// looked up, so this language never reaches cbm_lang_spec()/cbm_ts_language().
// Left as a zero spec.
// CBM_LANG_ASCIIDOC — AsciiDoc documents. No grammar row, as PDF: extraction
// reads the file itself (cbm_adoc_extract_document, doc_adoc.c). Zero spec.
// CBM_LANG_OBJECTSCRIPT_EXPORT — Studio Export XML. No grammar row: the
// pipeline transcodes Export XML to UDL (iris_export_xml.c) and re-extracts
// each class as CBM_LANG_OBJECTSCRIPT_UDL, so this language never reaches
internal/cbm/pdf/pdf.h
+63
View File
@@ -0,0 +1,63 @@
/*
* pdf.h — the text layer of a PDF, page by page.
*
* Reads what a PDF says in text (content-stream text operators decoded through
* the document's fonts) and nothing else: no rendering, no images, no OCR, no
* JavaScript, actions or URIs, nothing fetched, nothing executed. Input is
* untrusted: every read is bounds-checked, every recursion is depth-limited,
* every decoder is guarded against decompression bombs.
*
* Not read: encrypted documents (the result says so). Scanned pages have no
* text layer and come back empty.
*/
#ifndef CBM_PDF_H
#define CBM_PDF_H
#include "../arena.h"
#include <stdbool.h>
#include <stddef.h>
typedef enum {
CBM_PDF_OK = 0,
CBM_PDF_NOT_PDF, /* no "%PDF-" within the first 1024 bytes */
CBM_PDF_PARSE_FAILURE, /* no page could be found */
CBM_PDF_ENCRYPTED, /* an Encrypt dictionary: the text is not read */
CBM_PDF_NOMEM,
} cbm_pdf_status_t;
typedef struct {
const char *text; /* UTF-8, NUL-terminated, no NUL inside */
size_t len;
int glyphs; /* glyphs shown on the page */
int unmapped; /* glyphs without a Unicode value (U+FFFD in the text) */
} cbm_pdf_page_t;
/* How the document was read: counts of the repairs and limits that applied. */
typedef struct {
int xref_reconstructed; /* cross-reference rebuilt from a scan of the file */
int xref_offset_fixed; /* an xref section found near a wrong offset */
int stream_length_recovered; /* a stream's /Length was wrong */
int objstm_nested; /* opening an object stream needed an object of one */
int xref_streams; /* cross-reference streams decoded */
int form_rereads_cut; /* a form not read again: re-readings past 256 MiB */
int flate_recovered; /* a damaged Flate stream decoded in part */
int stream_guard_hits; /* a decoded stream cut at 512 MiB */
int filters_unsupported; /* a stream with a filter this reader does not decode */
int op_errors; /* content operators that could not be applied */
int page_errors; /* pages whose content stopped early */
} cbm_pdf_counts_t;
typedef struct {
cbm_pdf_status_t status;
cbm_pdf_page_t *pages;
int npages;
cbm_pdf_counts_t counts;
} cbm_pdf_result_t;
/* Read the text layer of data[0..len). Everything the result points to is
* allocated from `out` and lives as long as it. Returns false only when memory
* ran out (status CBM_PDF_NOMEM). */
bool cbm_pdf_extract(const unsigned char *data, size_t len, CBMArena *out, cbm_pdf_result_t *res);
#endif /* CBM_PDF_H */
internal/cbm/pdf/pdf_font.c
+1983
View File
File diff suppressed because it is too large Load Diff
internal/cbm/pdf/pdf_internal.h
+330
View File
@@ -0,0 +1,330 @@
/*
* pdf_internal.h — the PDF text-layer extractor's internals (pdf_*.c).
*
* Stages, each in its own file:
* pdf_obj.c lexer, values, the document (xref tables and streams, /Prev
* chains, reconstruction when offsets are broken), indirect
* objects, object streams, stream filters
* pdf_font.c glyph names, ToUnicode and encoding CMaps, simple fonts
* (base encodings, /Differences, built-in Type1 encodings,
* standard-14 widths, Type3), Type0/CID fonts, the TrueType
* cmap fallback
* pdf_text.c content streams (graphics and text state, text operators,
* Form XObjects, inline images skipped), layout (word and line
* breaks from glyph geometry), pages, the driver
* pdf_tables.c generated static tables
*
* The reader is a port of the field-tested prototype (field test E6): the
* same tokens, repairs, font rules and layout thresholds, so its text is the
* text that was measured. Where the prototype's cost could grow faster than
* the input (a search per object, a table walk per glyph), this port answers
* from an index built once, with the same result.
*/
#ifndef CBM_PDF_INTERNAL_H
#define CBM_PDF_INTERNAL_H
#include "pdf.h"
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
/* Layout geometry must come out bit for bit the same on every platform (and
* the same as the measured prototype): no fused multiply-add. GCC in ISO C
* mode never contracts; clang does unless told. */
#if defined(__clang__)
#pragma STDC FP_CONTRACT OFF
#endif
/* ── Tables (pdf_tables.c) ───────────────────────────────────────── */
typedef struct {
const char *name;
const char *utf8;
} pdf_glyph_t;
enum { PDF_STD_WIDTHS = 95 }; /* codes 0x20..0x7E */
extern const pdf_glyph_t PDF_GLYPHS[];
extern const int PDF_GLYPH_COUNT;
extern const char *const PDF_ENC_STANDARD[256];
extern const char *const PDF_ENC_WINANSI[256];
extern const char *const PDF_ENC_MACROMAN[256];
extern const char *const PDF_ENC_PDFDOC[256];
extern const char *const PDF_ENC_SYMBOL[256];
extern const unsigned short PDF_WIDTH_HELVETICA[PDF_STD_WIDTHS];
extern const unsigned short PDF_WIDTH_TIMES[PDF_STD_WIDTHS];
extern const char *const PDF_STD14[];
/* ── Limits: structure, never content ────────────────────────────── */
enum {
PDF_MAX_NESTING = 256, /* arrays/dicts inside one value */
PDF_MAX_RESOLVE = 32, /* reference chains */
PDF_MAX_FORM_DEPTH = 16, /* Form XObjects inside Form XObjects */
PDF_MAX_PAGE_DEPTH = 64, /* page-tree depth */
PDF_MAX_PREV = 4096, /* xref sections along /Prev */
PDF_HEADER_WINDOW = 1024, /* "%PDF-" must start within this many bytes */
};
/* A decoded stream above this size is cut and counted: a decompression bomb is
* no text layer, and real documents stay far below it. */
#define PDF_MAX_STREAM_OUT ((size_t)512 << 20)
/* A Form XObject's first reading in a document is always whole. Reading one
* again (the same content drawn elsewhere: a logo, a header) is counted, and
* beyond this many decoded bytes of such re-readings the document reads no
* form again (counted): forms drawing the next one many times, level under
* level, asked for (draws ^ depth) readings from a few kilobytes. */
#define PDF_MAX_FORM_REREAD ((size_t)256 << 20)
/* ── Integers from the file: saturated, never undefined ──────────── */
static inline int64_t pdf_sat_add(int64_t a, int64_t b) {
int64_t r;
if (__builtin_add_overflow(a, b, &r)) {
return b > 0 ? INT64_MAX : INT64_MIN;
}
return r;
}
/* Python's int() of a float: truncated toward zero; saturated here, NaN 0. */
static inline int64_t pdf_d2i(double x) {
if (x != x) {
return 0;
}
if (x >= 9223372036854775807.0) {
return INT64_MAX;
}
if (x <= -9223372036854775808.0) {
return INT64_MIN;
}
return (int64_t)x;
}
/* ── Values ──────────────────────────────────────────────────────── */
typedef enum {
PV_NULL = 0,
PV_BOOL,
PV_NUM,
PV_NAME,
PV_STR,
PV_ARR,
PV_DICT,
PV_REF,
PV_STREAM,
PV_KW,
} pdf_kind_t;
typedef struct pdf_val pdf_val_t;
typedef struct pdf_stream pdf_stream_t;
typedef struct {
const unsigned char *key; /* a name's bytes, NUL-terminated */
size_t klen;
pdf_val_t *val;
} pdf_kv_t;
struct pdf_val {
pdf_kind_t kind;
bool b;
bool is_int;
double num;
int64_t inum;
const unsigned char *s; /* PV_NAME, PV_STR, PV_KW (NUL-terminated copies) */
size_t n;
pdf_val_t **items; /* PV_ARR */
int count;
pdf_kv_t *kv; /* PV_DICT: sorted by key, one entry per key (the last written) */
int nkv;
int64_t ref_num; /* PV_REF */
int64_t ref_gen;
pdf_stream_t *stream; /* PV_STREAM */
};
struct pdf_stream {
pdf_val_t *dict;
const unsigned char *raw;
size_t raw_len;
bool done;
const unsigned char *decoded; /* NULL: nothing decodable (image, unsupported) */
size_t dec_len;
};
/* ── Lexer ───────────────────────────────────────────────────────── */
typedef enum {
PT_EOF = 0,
PT_NUM,
PT_NAME,
PT_STR,
PT_DS, /* << */
PT_DE, /* >> */
PT_AS, /* [ */
PT_AE, /* ] */
PT_BS, /* { */
PT_BE, /* } */
PT_KW,
} pdf_tok_kind_t;
typedef struct {
pdf_tok_kind_t kind;
bool is_int;
double num;
int64_t inum;
const unsigned char *s; /* NAME, STR: arena copies; KW: points into the data */
size_t n;
} pdf_tok_t;
typedef struct pdf_doc pdf_doc_t;
/* The next token of data[*pos..len); *pos moves past it. */
void pdf_lex(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos, pdf_tok_t *t);
/* The value that starts with token t. NULL: out of memory or nested too deep. */
pdf_val_t *pdf_parse_value(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos,
const pdf_tok_t *t, int depth);
/* An array operand of a content stream: no references, an operator ends it. */
pdf_val_t *pdf_parse_content_array(pdf_doc_t *d, const unsigned char *data, size_t len, size_t *pos,
int depth);
bool pdf_kw_is(const pdf_tok_t *t, const char *kw);
/* ── Integer maps (open addressing, owned by the document) ───────── */
typedef struct {
uint64_t *keys;
int64_t *vals;
uint32_t cap; /* a power of two, or 0 */
uint32_t n;
} pdf_map_t;
/* -1 when absent. */
int64_t pdf_map_get(const pdf_map_t *m, uint64_t key);
bool pdf_map_put(pdf_map_t *m, uint64_t key, int64_t val);
void pdf_map_free(pdf_map_t *m);
/* ── Document ────────────────────────────────────────────────────── */
typedef struct pdf_objstm pdf_objstm_t;
typedef struct {
int64_t num;
uint8_t type; /* 0 free, 1 at an offset, 2 inside an object stream */
int64_t a; /* type 1: offset; type 2: the object stream's number */
int64_t b; /* type 1: generation; type 2: index in the stream */
pdf_val_t *obj;
bool cached;
bool loading;
bool not_length; /* read as a stream's /Length, it was a stream: no length */
pdf_objstm_t *stm; /* this object as a decoded object stream */
} pdf_xent_t;
typedef struct {
pdf_xent_t *v;
int n;
int cap;
pdf_map_t by_num; /* object number -> index in v */
} pdf_xref_t;
/* One "N G obj" header found by scanning the whole file. */
typedef struct {
int64_t num;
int64_t gen;
size_t off; /* where N starts */
uint8_t num_digits;
uint8_t gen_digits;
bool canonical; /* both written without leading zeros */
bool reg_after; /* "obj" runs on into a regular character */
} pdf_hdr_t;
typedef struct {
int64_t num;
int idx;
} pdf_numidx_t;
typedef struct pdf_font pdf_font_t;
struct pdf_doc {
const unsigned char *data;
size_t len;
size_t hdr; /* offset of "%PDF-" */
CBMArena *a; /* the document's values, fonts, CMaps */
CBMArena *cur; /* where the lexer and the value parser allocate (a, or a
* content stream's operand arena) */
bool peek; /* lex without copying names or strings */
bool nomem; /* an allocation failed: the result is CBM_PDF_NOMEM */
pdf_xref_t xref;
pdf_val_t *trailer; /* a dict */
bool reconstructed;
int objstm_opening; /* object streams being opened, one inside the other */
/* every "N G obj" in file order, built on first need */
pdf_hdr_t *hdrs;
int nhdrs;
bool hdrs_built;
pdf_numidx_t *hdr_sorted; /* (num, index) sorted, for the per-object fallback */
/* blocks from the memory core, freed with the document */
void **owned;
int nowned;
int owned_cap;
/* an arena for token values that end with one CMap's parse */
CBMArena scratch_arena;
bool scratch_arena_live;
/* a scratch buffer for string and stream assembly */
unsigned char *scratch;
size_t scratch_cap;
/* fonts by reference (num << 20 | gen) or by inline dict address */
pdf_map_t fonts;
pdf_font_t **font_list;
int nfonts;
int font_cap;
cbm_pdf_counts_t *counts;
};
pdf_val_t *pdf_get(pdf_doc_t *d, int64_t num);
pdf_val_t *pdf_resolve(pdf_doc_t *d, pdf_val_t *v);
/* dict[key] as written (a reference stays a reference); NULL when absent, null,
* or the value is no dict. */
pdf_val_t *pdf_dget_raw(const pdf_val_t *dict, const char *key);
/* Whether dict has key at all (a null value counts). */
bool pdf_dhas(const pdf_val_t *dict, const char *key);
/* dict[key] as written, a null value included; NULL only when absent. */
pdf_val_t *pdf_dfind(const pdf_val_t *dict, const char *key);
pdf_val_t *pdf_dfind_n(const pdf_val_t *dict, const unsigned char *key, size_t n);
/* dict[key] resolved; NULL when absent or null. */
pdf_val_t *pdf_dget(pdf_doc_t *d, const pdf_val_t *dict, const char *key);
bool pdf_is_name(const pdf_val_t *v, const char *name);
bool pdf_is_num(const pdf_val_t *v);
/* A number's value; 0 for anything else. */
double pdf_num(const pdf_val_t *v);
/* The decoded data of a stream; NULL when it has none. */
const unsigned char *pdf_stream_decode(pdf_doc_t *d, pdf_stream_t *st, size_t *len);
bool pdf_doc_open(pdf_doc_t *d);
void pdf_doc_close(pdf_doc_t *d);
/* Memory: arena values, and big blocks the document owns. */
void *pdf_alloc(pdf_doc_t *d, size_t n);
void *pdf_calloc(pdf_doc_t *d, size_t n);
void *pdf_big(pdf_doc_t *d, size_t n); /* owned, freed with the document */
bool pdf_own(pdf_doc_t *d, void *block); /* hand a memory-core block to the document */
bool pdf_scratch(pdf_doc_t *d, size_t need); /* d->scratch holds at least `need` bytes */
/* ── Fonts (pdf_font.c) ──────────────────────────────────────────── */
typedef struct {
const char *utf8; /* NULL: unmapped; "" maps to nothing; may point into inl */
double width; /* advance in text-space units per unit of font size */
bool space; /* the single-byte code 32 */
char inl[16]; /* a short mapping made for this glyph */
} pdf_glyph_out_t;
/* The font resources[/Font][name] names; NULL when there is none. Cached. */
pdf_font_t *pdf_font_for(pdf_doc_t *d, pdf_val_t *resources, const pdf_val_t *name);
/* Decode shown bytes into glyphs, out[0..*n) (out holds at least len entries). */
void pdf_font_decode(pdf_doc_t *d, pdf_font_t *f, const unsigned char *s, size_t len,
pdf_glyph_out_t *out, size_t *n);
/* The text-space size of one em per unit of font size. */
double pdf_font_em(const pdf_font_t *f);
/* A glyph name's Unicode value as UTF-8 (arena or static); NULL when unknown. */
const char *pdf_glyph_to_utf8(pdf_doc_t *d, const unsigned char *name, size_t n);
#endif /* CBM_PDF_INTERNAL_H */
internal/cbm/pdf/pdf_obj.c
+2771
View File
File diff suppressed because it is too large Load Diff
internal/cbm/pdf/pdf_tables.c
+1624
View File
File diff suppressed because it is too large Load Diff
internal/cbm/pdf/pdf_text.c
+1054
View File
File diff suppressed because it is too large Load Diff
internal/cbm/pdf/pdf_unicode.c
+3126
View File
File diff suppressed because it is too large Load Diff
internal/cbm/pdf/pdf_unicode.h
+103
View File
@@ -0,0 +1,103 @@
/*
* pdf_unicode.h — the Unicode facts the PDF mention scanner needs, exactly
* as CPython answers them (pdf_unicode.c is generated from CPython's
* unicodedata; the field-tested scanner ran on CPython's str and re).
*/
#ifndef CBM_PDF_UNICODE_H
#define CBM_PDF_UNICODE_H
#include <stdbool.h>
#include <stdint.h>
typedef struct {
uint32_t cp;
uint16_t off; /* into PDF_NFKC_SEQ */
uint8_t len;
} pdf_nfkc_t;
typedef struct {
uint32_t first;
uint32_t second;
uint32_t composite;
} pdf_ucomp_t;
typedef struct {
uint32_t lo;
uint32_t hi;
} pdf_urange_t;
extern const pdf_nfkc_t PDF_NFKC[]; /* sorted by cp: NFKC of one code point where it differs */
extern const int PDF_NFKC_COUNT;
extern const uint32_t PDF_NFKC_SEQ[];
extern const pdf_ucomp_t PDF_UCOMP[]; /* sorted by (first, second): canonical compositions */
extern const int PDF_UCOMP_COUNT;
extern const pdf_urange_t PDF_U_SPACE[]; /* str.isspace */
extern const int PDF_U_SPACE_COUNT;
extern const pdf_urange_t PDF_U_WORD[]; /* re's \w: str.isalnum or '_' */
extern const int PDF_U_WORD_COUNT;
extern const pdf_urange_t PDF_U_ALPHA[]; /* str.isalpha */
extern const int PDF_U_ALPHA_COUNT;
extern const pdf_urange_t PDF_U_UPPER[]; /* one-character str.isupper */
extern const int PDF_U_UPPER_COUNT;
extern const pdf_urange_t PDF_U_LOWER[]; /* one-character str.islower */
extern const int PDF_U_LOWER_COUNT;
extern const pdf_urange_t PDF_U_TITLE[]; /* titlecase letters (neither upper nor lower) */
extern const int PDF_U_TITLE_COUNT;
static inline bool pdf_u_in(const pdf_urange_t *r, int n, uint32_t cp) {
int lo = 0;
int hi = n - 1;
while (lo <= hi) {
int mid = lo + (hi - lo) / 2;
if (r[mid].hi < cp) {
lo = mid + 1;
} else if (r[mid].lo > cp) {
hi = mid - 1;
} else {
return true;
}
}
return false;
}
static inline bool pdf_u_space(uint32_t c) {
if (c < 0x80) {
return c == ' ' || (c >= 0x09 && c <= 0x0D) || (c >= 0x1C && c <= 0x1F);
}
return pdf_u_in(PDF_U_SPACE, PDF_U_SPACE_COUNT, c);
}
static inline bool pdf_u_word(uint32_t c) {
if (c < 0x80) {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
c == '_';
}
return pdf_u_in(PDF_U_WORD, PDF_U_WORD_COUNT, c);
}
static inline bool pdf_u_alpha(uint32_t c) {
if (c < 0x80) {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
}
return pdf_u_in(PDF_U_ALPHA, PDF_U_ALPHA_COUNT, c);
}
static inline bool pdf_u_upper(uint32_t c) {
if (c < 0x80) {
return c >= 'A' && c <= 'Z';
}
return pdf_u_in(PDF_U_UPPER, PDF_U_UPPER_COUNT, c);
}
static inline bool pdf_u_lower(uint32_t c) {
if (c < 0x80) {
return c >= 'a' && c <= 'z';
}
return pdf_u_in(PDF_U_LOWER, PDF_U_LOWER_COUNT, c);
}
static inline bool pdf_u_title(uint32_t c) {
return c >= 0x80 && pdf_u_in(PDF_U_TITLE, PDF_U_TITLE_COUNT, c);
}
#endif /* CBM_PDF_UNICODE_H */
internal/cbm/result_compact.c
+1
View File
@@ -276,6 +276,7 @@ static void cr_walk_def(cr_ctx_t *c, CBMDefinition *d) {
cr_str(c, &d->http_client);
cr_str(c, &d->http_base_url);
cr_str(c, &d->variants);
cr_list(c, &d->extra_props);
cr_str(c, &d->decl_namespace);
}
scripts/ci/generate-sbom.py
+3
View File
@@ -49,6 +49,9 @@ sbom = {
{"SPDXID": "SPDXRef-Package-simplecpp", "name": "simplecpp", "versionInfo": "1.x", "licenseDeclared": "0BSD", "downloadLocation": "https://github.com/danmar/simplecpp", "filesAnalyzed": False},
{"SPDXID": "SPDXRef-Package-verstable", "name": "verstable", "versionInfo": "2.2.1", "licenseDeclared": "MIT", "downloadLocation": "https://github.com/JacksonAllan/Verstable", "filesAnalyzed": False},
{"SPDXID": "SPDXRef-Package-wyhash", "name": "wyhash", "versionInfo": "final-4.3", "licenseDeclared": "Unlicense", "downloadLocation": "https://github.com/wangyi-fudan/wyhash", "filesAnalyzed": False},
{"SPDXID": "SPDXRef-Package-adobe-glyph-list", "name": "adobe-glyph-list", "versionInfo": "2.0", "licenseDeclared": "BSD-3-Clause", "downloadLocation": "https://github.com/adobe-type-tools/agl-aglfn", "filesAnalyzed": False, "comment": "Glyph-name subset in internal/cbm/pdf/pdf_tables.c (PDF text layer); see THIRD_PARTY.md"},
{"SPDXID": "SPDXRef-Package-adobe-core14-metrics", "name": "adobe-core14-afm-metrics", "versionInfo": "1997", "licenseDeclared": "NOASSERTION", "downloadLocation": "NOASSERTION", "filesAnalyzed": False, "comment": "Printable-ASCII advance widths of Helvetica and Times-Roman in internal/cbm/pdf/pdf_tables.c; (c) 1985-1997 Adobe Systems Incorporated; the AFM files are not distributed; see THIRD_PARTY.md"},
{"SPDXID": "SPDXRef-Package-unicode-ucd", "name": "unicode-character-database", "versionInfo": "16.0.0", "licenseDeclared": "Unicode-3.0", "downloadLocation": "https://www.unicode.org/Public/16.0.0/ucd/", "filesAnalyzed": False, "comment": "NFKC, compositions and character classes in internal/cbm/pdf/pdf_unicode.c, generated by scripts/gen-pdf-unicode.py; see THIRD_PARTY.md"},
{"SPDXID": "SPDXRef-Package-nomic-embed-code", "name": "nomic-embed-code-token-embeddings", "versionInfo": "1.0", "licenseDeclared": "Apache-2.0", "downloadLocation": "https://huggingface.co/nomic-ai/nomic-embed-code", "filesAnalyzed": False, "comment": "Derived int8 token embeddings; see vendored/nomic/NOTICE"},
{"SPDXID": "SPDXRef-Package-tree-sitter-grammars", "name": "tree-sitter-grammars-aggregate", "versionInfo": f"{n_grammars}-grammars", "licenseDeclared": "MIT", "downloadLocation": "NOASSERTION", "filesAnalyzed": False, "comment": f"Aggregate of {n_grammars} vendored tree-sitter grammars, compiled statically. Predominantly MIT; non-MIT families present include CC0-1.0 (clojure, fennel), Apache-2.0 (elixir, erlang, gleam, hcl, ini, jinja2, just, pkl, sway, wit), ISC (pine, templ), Unlicense (fish); first-party grammars (c) DeusData, MIT. This summary is non-exhaustive; the authoritative complete per-grammar license list is internal/cbm/vendored/grammars/MANIFEST.md, and full license texts ship in THIRD_PARTY_NOTICES.md inside each archive."},
],
scripts/gen-pdf-unicode.py
+103
View File
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
"""Generate internal/cbm/pdf/pdf_unicode.c: the Unicode facts the PDF mention
scanner needs, exactly as CPython sees them (the field test's scanner ran on
CPython's str and re).
usage: scripts/gen-pdf-unicode.py internal/cbm/pdf/pdf_unicode.c (then clang-format -i it)
The output depends on the CPython version (its Unicode data): the committed
tables were made with CPython 3.14.6 (Unicode 16.0.0).
"""
import sys
import unicodedata
MAXCP = 0x110000
def is_surrogate(c):
return 0xD800 <= c <= 0xDFFF
def ranges(pred):
out = []
start = None
for c in range(MAXCP):
ok = not is_surrogate(c) and pred(chr(c))
if ok and start is None:
start = c
elif not ok and start is not None:
out.append((start, c - 1))
start = None
if start is not None:
out.append((start, MAXCP - 1))
return out
def emit_ranges(f, name, rs):
f.write("const pdf_urange_t %s[] = {\n" % name)
for i in range(0, len(rs), 4):
f.write(" " + " ".join("{0x%X, 0x%X}," % r for r in rs[i:i + 4]) + "\n")
f.write("};\nconst int %s_COUNT = (int)(sizeof(%s) / sizeof(%s[0]));\n\n" % (name, name, name))
def main():
out = sys.argv[1]
# per-character NFKC
nfkc = []
seq = []
for c in range(MAXCP):
if is_surrogate(c):
continue
ch = chr(c)
m = unicodedata.normalize("NFKC", ch)
if m != ch:
nfkc.append((c, len(seq), len(m)))
seq.extend(ord(x) for x in m)
# canonical composition pairs: a + b -> c when NFC composes them
comp = []
for c in range(MAXCP):
if is_surrogate(c):
continue
d = unicodedata.decomposition(chr(c))
if not d or d.startswith("<"):
continue
parts = [int(x, 16) for x in d.split()]
if len(parts) != 2:
continue
a, b = parts
if unicodedata.normalize("NFC", chr(a) + chr(b)) == chr(c):
comp.append((a, b, c))
comp.sort()
with open(out, "w", encoding="utf-8") as f:
f.write("/*\n * pdf_unicode.c -- Unicode %s facts for the PDF mention scanner (generated).\n"
" *\n * The field test's scanner ran on CPython's str and re; these tables are what\n"
" * CPython %s answers: NFKC of each code point, the canonical compositions,\n"
" * str.isspace, the re module's \\w (str.isalnum or '_'), str.isalpha, and the\n"
" * one-character str.isupper / str.islower / titlecase. Generated by\n"
" * scripts/gen-pdf-unicode.py; do not edit by hand. Data of the Unicode\n"
" * Character Database, (c) Unicode, Inc., Unicode License v3\n"
" * (THIRD_PARTY.md).\n */\n"
% (unicodedata.unidata_version, sys.version.split()[0]))
f.write("#include \"pdf_unicode.h\"\n\n")
f.write("const pdf_nfkc_t PDF_NFKC[] = {\n")
for i in range(0, len(nfkc), 4):
f.write(" " + " ".join("{0x%X, %d, %d}," % e for e in nfkc[i:i + 4]) + "\n")
f.write("};\nconst int PDF_NFKC_COUNT = (int)(sizeof(PDF_NFKC) / sizeof(PDF_NFKC[0]));\n\n")
f.write("const uint32_t PDF_NFKC_SEQ[] = {\n")
for i in range(0, len(seq), 10):
f.write(" " + " ".join("0x%X," % x for x in seq[i:i + 10]) + "\n")
f.write("};\n\n")
f.write("const pdf_ucomp_t PDF_UCOMP[] = {\n")
for i in range(0, len(comp), 3):
f.write(" " + " ".join("{0x%X, 0x%X, 0x%X}," % e for e in comp[i:i + 3]) + "\n")
f.write("};\nconst int PDF_UCOMP_COUNT = (int)(sizeof(PDF_UCOMP) / sizeof(PDF_UCOMP[0]));\n\n")
emit_ranges(f, "PDF_U_SPACE", ranges(lambda c: c.isspace()))
emit_ranges(f, "PDF_U_WORD", ranges(lambda c: c.isalnum() or c == "_"))
emit_ranges(f, "PDF_U_ALPHA", ranges(lambda c: c.isalpha()))
emit_ranges(f, "PDF_U_UPPER", ranges(lambda c: c.isupper()))
emit_ranges(f, "PDF_U_LOWER", ranges(lambda c: c.islower()))
emit_ranges(f, "PDF_U_TITLE", ranges(lambda c: c.istitle() and not c.isupper()))
print("nfkc", len(nfkc), "seq", len(seq), "comp", len(comp))
if __name__ == "__main__":
main()
src/discover/discover.c
+160 -3
View File
@@ -16,6 +16,7 @@
#include "foundation/constants.h"
#include "foundation/compat_fs.h"
#include "foundation/limits.h"
#include "foundation/mem_core.h"
#include "foundation/workspace.h"
#include "foundation/platform.h"
#ifdef _WIN32
@@ -888,13 +889,159 @@ static int safe_stat(const char *abs_path, struct stat *st, bool *is_symlink) {
#endif
}
/* ── Sphinx documentation sets ────────────────────────────────────
* A directory with a conf.py is a Sphinx documentation set, and its
* `source_suffix` says which file suffixes are reStructuredText (Django's
* docs are `.txt`). A file of such a suffix in the set's subtree, of no
* language by its name, is reST; a nested conf.py starts its own set. conf.py
* is read as text (the first SPHINX_CONF_READ bytes) and never run. */
enum {
SPHINX_SUFFIXES = 4, /* extra reST suffixes one set declares */
SPHINX_SUFFIX_LEN = 16, /* ".txt", ".rest": a longer one is no suffix */
SPHINX_CONF_READ = 65536,
};
typedef struct {
char ext[SPHINX_SUFFIXES][SPHINX_SUFFIX_LEN];
int count;
} sphinx_suffixes_t;
/* The quoted string at s[i] ('...' or "..."): its span; false otherwise. */
static bool sphinx_quoted(const char *s, size_t n, size_t i, size_t *a, size_t *b) {
if (i >= n || (s[i] != '\'' && s[i] != '"')) {
return false;
}
const char *close = memchr(s + i + SKIP_ONE, s[i], n - i - SKIP_ONE);
if (!close) {
return false;
}
*a = i + SKIP_ONE;
*b = (size_t)(close - s);
return true;
}
static void sphinx_add(sphinx_suffixes_t *out, const char *s, size_t n) {
if (n < PAIR_LEN || n >= SPHINX_SUFFIX_LEN || s[0] != '.' || out->count >= SPHINX_SUFFIXES ||
(n == strlen(".rst") && memcmp(s, ".rst", n) == 0)) {
return;
}
for (size_t i = SKIP_ONE; i < n; i++) {
if (!isalnum((unsigned char)s[i]) && s[i] != '_' && s[i] != '-') {
return;
}
}
memcpy(out->ext[out->count], s, n);
out->ext[out->count][n] = '\0';
out->count++;
}
/* `source_suffix = '.txt'`, `= ['.rst', '.txt']` or
* `= {'.rst': 'restructuredtext', '.txt': 'restructuredtext'}` (a dict
* entry counts only when it maps to restructuredtext). */
static void sphinx_parse(const char *s, size_t n, sphinx_suffixes_t *out) {
static const char key[] = "source_suffix";
size_t kl = strlen(key);
for (size_t line = 0; line < n;) {
const char *nl = memchr(s + line, '\n', n - line);
size_t end = nl ? (size_t)(nl - s) : n;
size_t i = line + kl;
if (end >= i && memcmp(s + line, key, kl) == 0) {
while (i < n && (s[i] == ' ' || s[i] == '\t')) {
i++;
}
if (i < n && s[i] == '=') {
i++;
while (i < n && (s[i] == ' ' || s[i] == '\t')) {
i++;
}
char open = i < n ? s[i] : '\0';
char close = '\0';
if (open == '[' || open == '(' || open == '{') {
close = open == '[' ? ']' : (open == '(' ? ')' : '}');
}
/* the value: up to its closing bracket, or this line */
size_t stop = end;
if (close) {
const char *c = memchr(s + i, close, n - i);
stop = c ? (size_t)(c - s) : end;
}
size_t k = i;
while (k < stop) {
size_t a;
size_t b;
if (!sphinx_quoted(s, stop, k, &a, &b)) {
k++;
continue;
}
k = b + SKIP_ONE;
if (open != '{') {
sphinx_add(out, s + a, b - a);
continue;
}
while (k < stop && (s[k] == ' ' || s[k] == ':')) {
k++;
}
size_t va;
size_t vb;
if (sphinx_quoted(s, stop, k, &va, &vb)) {
if (vb - va == strlen("restructuredtext") &&
memcmp(s + va, "restructuredtext", vb - va) == 0) {
sphinx_add(out, s + a, b - a);
}
k = vb + SKIP_ONE;
}
}
return; /* the first assignment */
}
}
line = end + SKIP_ONE;
}
}
/* The documentation set this directory starts, if it has a conf.py. */
static bool sphinx_load(const char *dir, sphinx_suffixes_t *out) {
char conf[CBM_SZ_4K];
if (snprintf(conf, sizeof(conf), "%s/conf.py", dir) >= (int)sizeof(conf)) {
return false;
}
struct stat st;
if (wide_stat(conf, &st) != 0 || !S_ISREG(st.st_mode)) {
return false;
}
FILE *f = cbm_fopen(conf, "rb");
if (!f) {
return false;
}
char *buf = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, SPHINX_CONF_READ);
size_t n = buf ? fread(buf, SKIP_ONE, SPHINX_CONF_READ, f) : 0;
(void)fclose(f);
memset(out, 0, sizeof(*out));
if (buf) {
sphinx_parse(buf, n, out);
}
cbm_free(CBM_MEM_CLASS_OTHER, buf);
return true;
}
static bool sphinx_is_rst(const sphinx_suffixes_t *set, const char *name) {
size_t nl = strlen(name);
for (int i = 0; set && i < set->count; i++) {
size_t el = strlen(set->ext[i]);
if (nl > el && strcmp(name + nl - el, set->ext[i]) == 0) {
return true;
}
}
return false;
}
/* Process a single regular file entry during directory walk. */
static void walk_dir_process_file(const char *abs_path, const char *rel_path, const char *name,
const cbm_discover_opts_t *opts,
const gitignore_link_t *ignore_chain,
const cbm_gitignore_t *global_gi,
const cbm_gitignore_t *cbmignore, const struct stat *st,
file_list_t *out) {
const cbm_gitignore_t *cbmignore, const sphinx_suffixes_t *sphinx,
const struct stat *st, file_list_t *out) {
const char *skip_reason =
file_skip_reason(name, rel_path, opts, ignore_chain, global_gi, cbmignore, st->st_size);
if (skip_reason) {
@@ -906,6 +1053,9 @@ static void walk_dir_process_file(const char *abs_path, const char *rel_path, co
return;
}
CBMLanguage lang = detect_file_language(name, abs_path);
if (lang == CBM_LANG_COUNT && sphinx_is_rst(sphinx, name)) {
lang = CBM_LANG_RST; /* its documentation set's source_suffix */
}
if (lang == CBM_LANG_COUNT) {
return;
}
@@ -916,6 +1066,7 @@ typedef struct {
char dir[CBM_SZ_4K];
char prefix[CBM_SZ_4K];
const gitignore_link_t *ignore_chain; /* deepest .gitignore governing this dir */
sphinx_suffixes_t sphinx; /* the documentation set's extra reST suffixes */
} walk_frame_t;
/* Initial capacity only — the stack grows on demand. A single directory can
* hold more pending sibling frames than any fixed cap (dotnet/runtime has 855
@@ -973,6 +1124,7 @@ static void walk_push_subdir(walk_stack_t *ws, const char *abs_path, const char
return;
}
slot->ignore_chain = parent->ignore_chain;
slot->sphinx = parent->sphinx;
ws->top++;
}
@@ -1021,7 +1173,7 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram
}
} else if (S_ISREG(st.st_mode)) {
walk_dir_process_file(abs_path, rel_path, entry->name, opts, frame->ignore_chain, global_gi,
cbmignore, &st, out);
cbmignore, &frame->sphinx, &st, out);
}
}
@@ -1115,6 +1267,11 @@ static void walk_dir(const char *dir_path, const char *rel_prefix, const cbm_dis
frame.ignore_chain = link;
}
sphinx_suffixes_t set;
if (sphinx_load(frame.dir, &set)) {
frame.sphinx = set; /* a (nested) documentation set starts here */
}
cbm_dir_t *d = cbm_opendir(frame.dir);
if (!d) {
if (out->count_only) {
src/discover/language.c
+6
View File
@@ -618,6 +618,10 @@ static const ext_entry_t EXT_TABLE[] = {
{".clsp", CBM_LANG_CHIALISP},
{".clib", CBM_LANG_CHIALISP},
{".clinc", CBM_LANG_CHIALISP},
{".pdf", CBM_LANG_PDF},
/* AsciiDoc; `.asc` is left out: it is as often an ASCII-armored signature */
{".adoc", CBM_LANG_ASCIIDOC},
{".asciidoc", CBM_LANG_ASCIIDOC},
/* Slang */
{".slang", CBM_LANG_SLANG},
@@ -829,6 +833,8 @@ static const char *LANG_NAMES[CBM_LANG_COUNT] = {
[CBM_LANG_NIM] = "Nim",
[CBM_LANG_SCHEME] = "Scheme",
[CBM_LANG_CHIALISP] = "Chialisp",
[CBM_LANG_PDF] = "PDF",
[CBM_LANG_ASCIIDOC] = "AsciiDoc",
[CBM_LANG_FENNEL] = "Fennel",
[CBM_LANG_FISH] = "Fish",
[CBM_LANG_AWK] = "AWK",
src/discover/userconfig.c
+2
View File
@@ -160,6 +160,8 @@ static const lang_name_entry_t LANG_NAME_TABLE[] = {
{"mojo", CBM_LANG_MOJO},
{"plsql", CBM_LANG_PLSQL},
{"chialisp", CBM_LANG_CHIALISP},
{"pdf", CBM_LANG_PDF},
{"asciidoc", CBM_LANG_ASCIIDOC},
{"lean", CBM_LANG_LEAN},
{"form", CBM_LANG_FORM},
{"magma", CBM_LANG_MAGMA},
src/mcp/mcp.c
+38 -2
View File
@@ -12582,6 +12582,36 @@ static char *resolve_path_owner(const char *path, char *derived) {
return owner;
}
/* A PDF is read as its text layer at index time (doc_pdf.c); its bytes are no
* source text. Its page Sections carry the page's text as their docstring. */
static bool snippet_is_pdf(const char *file_path) {
size_t n = file_path ? strlen(file_path) : 0;
return n >= 4 && file_path[n - 4] == '.' && (file_path[n - 3] | 0x20) == 'p' &&
(file_path[n - 2] | 0x20) == 'd' && (file_path[n - 1] | 0x20) == 'f';
}
static char *snippet_pdf_text(const cbm_node_t *node) {
if (!node->properties_json) {
return NULL;
}
yyjson_doc *pd = yyjson_read(node->properties_json, strlen(node->properties_json), 0);
if (!pd) {
return NULL;
}
yyjson_val *ds = yyjson_obj_get(yyjson_doc_get_root(pd), "docstring");
const char *text = yyjson_is_str(ds) ? yyjson_get_str(ds) : NULL;
char *out = NULL;
if (text && text[0]) {
/* the same builder read_file_lines returns its text from */
cbm_sb_t sb;
cbm_sb_init(&sb);
cbm_sb_append_n(&sb, text, strlen(text));
out = cbm_sb_finish(&sb);
}
yyjson_doc_free(pd);
return out;
}
static char *resolve_snippet_source(const char *root_path, const char *file_path, int start,
int end, char **out_abs_path) {
*out_abs_path = NULL;
@@ -12932,8 +12962,14 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node,
snippet_clipped = true;
}
char *abs_path = NULL;
char *source =
outline ? NULL : resolve_snippet_source(root_path, node->file_path, start, end, &abs_path);
char *source = NULL;
if (outline) {
source = NULL;
} else if (snippet_is_pdf(node->file_path)) {
source = snippet_pdf_text(node); /* never the file's bytes */
} else {
source = resolve_snippet_source(root_path, node->file_path, start, end, &abs_path);
}
yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL);
yyjson_mut_val *root_obj = yyjson_mut_obj(doc);
src/pipeline/doc_links.c
+39 -11
View File
@@ -45,7 +45,8 @@ const char *cbm_doclink_reason_name(int reason) {
/* One pointer per language with a resolver: the one line a language leg adds
* to this file. */
static const cbm_doclink_resolver_t *const DOCLINK_RESOLVERS[] = {
&cbm_doclink_cs_resolver,
&cbm_doclink_cs_resolver, &cbm_doclink_md_resolver, &cbm_doclink_pdf_resolver,
&cbm_doclink_rst_resolver, &cbm_doclink_adoc_resolver,
};
enum { DOCLINK_RESOLVER_COUNT = sizeof(DOCLINK_RESOLVERS) / sizeof(DOCLINK_RESOLVERS[0]) };
@@ -89,6 +90,10 @@ static const cbm_doclink_resolver_t *resolver_of_scope(const char *scope) {
/* ── Incremental scope rules ─────────────────────────────────────── */
bool cbm_doclinks_binds_lines(CBMLanguage lang) {
return lang == CBM_LANG_MARKDOWN || lang == CBM_LANG_RST || lang == CBM_LANG_ASCIIDOC;
}
bool cbm_doclinks_is_scope_input(const char *rel_path) {
for (int i = 0; rel_path && i < DOCLINK_RESOLVER_COUNT; i++) {
if (DOCLINK_RESOLVERS[i]->scope_input && DOCLINK_RESOLVERS[i]->scope_input(rel_path)) {
@@ -328,6 +333,9 @@ typedef struct {
int64_t src;
int64_t tgt;
uint32_t line;
uint32_t target_first; /* the target lines the reference names; 0: none */
uint32_t target_last;
const char *via;
uint16_t syntax;
bool exact;
} doclink_mention_t;
@@ -335,6 +343,10 @@ typedef struct {
static int mention_cmp(const void *a, const void *b) {
const doclink_mention_t *x = (const doclink_mention_t *)a;
const doclink_mention_t *y = (const doclink_mention_t *)b;
int edge = strcmp(cbm_doclink_syntax_edge(x->syntax), cbm_doclink_syntax_edge(y->syntax));
if (edge != 0) {
return edge;
}
if (x->src != y->src) {
return x->src < y->src ? -1 : 1;
}
@@ -368,7 +380,8 @@ void cbm_doclinks_test_fail_edge_insert_after(int nth) {
#endif
/* Insert one MENTIONS edge; 0 when it could not be stored. */
static int64_t doclinks_insert_edge(cbm_gbuf_t *gb, int64_t src, int64_t tgt, const char *props) {
static int64_t doclinks_insert_edge(cbm_gbuf_t *gb, int64_t src, int64_t tgt, const char *type,
const char *props) {
#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
int n = atomic_load(&doclinks_test_edge_fail_after);
while (n > 0) {
@@ -380,10 +393,10 @@ static int64_t doclinks_insert_edge(cbm_gbuf_t *gb, int64_t src, int64_t tgt, co
}
}
#endif
return cbm_gbuf_insert_edge(gb, src, tgt, "MENTIONS", props);
return cbm_gbuf_insert_edge(gb, src, tgt, type, props);
}
/* Emit one MENTIONS edge per (source, target): first line, its syntax, the
/* Emit one edge per (edge type, source, target): first line, its syntax, the
* mention count, and tier exact when any mention bound exactly. A failed
* insert fails the layer: the edge is not there. */
static void emit_mentions(cbm_doclinks_t *dl, const char *rel, doclink_mention_t *m, int n,
@@ -393,17 +406,24 @@ static void emit_mentions(cbm_doclinks_t *dl, const char *rel, doclink_mention_t
while (i < n) {
int j = i;
bool exact = false;
while (j < n && m[j].src == m[i].src && m[j].tgt == m[i].tgt) {
const char *type = cbm_doclink_syntax_edge(m[i].syntax);
while (j < n && m[j].src == m[i].src && m[j].tgt == m[i].tgt &&
strcmp(cbm_doclink_syntax_edge(m[j].syntax), type) == 0) {
exact = exact || m[j].exact;
j++;
}
char props[CBM_SZ_256];
char lines[CBM_SZ_64] = "";
if (m[i].target_first > 0) {
snprintf(lines, sizeof(lines), ",\"target_lines\":[%u,%u]", m[i].target_first,
m[i].target_last);
}
snprintf(props, sizeof(props),
"{\"via\":\"doc_comment\",\"syntax\":\"%s\",\"tier\":\"%s\",\"line\":%u,"
"\"count\":%d}",
cbm_doclink_syntax_name(m[i].syntax), exact ? "exact" : "unique", m[i].line,
j - i);
if (doclinks_insert_edge(edge_out, m[i].src, m[i].tgt, props) == 0) {
"{\"via\":\"%s\",\"syntax\":\"%s\",\"tier\":\"%s\",\"line\":%u,"
"\"count\":%d%s}",
m[i].via, cbm_doclink_syntax_name(m[i].syntax), exact ? "exact" : "unique",
m[i].line, j - i, lines);
if (doclinks_insert_edge(edge_out, m[i].src, m[i].tgt, type, props) == 0) {
mark_failed(dl, rel, "alloc"); /* an edge that is not there is not counted */
} else {
atomic_fetch_add_explicit(&dl->edges, 1, memory_order_relaxed);
@@ -446,7 +466,12 @@ void cbm_doclinks_resolve_file(cbm_doclinks_t *dl, int file_idx, const CBMFileRe
* file-level doc resolves */
const cbm_gbuf_node_t *file_node = NULL;
bool file_node_looked_up = false;
void *state = (R && index && R->file_begin) ? R->file_begin(index, file_idx) : NULL;
void *state = NULL;
if (R && index && R->file_prepare) {
state = R->file_prepare(index, file_idx, result->doc_links.items, n, graph);
} else if (R && index && R->file_begin) {
state = R->file_begin(index, file_idx);
}
for (int i = 0; i < n; i++) {
const CBMDocLink *link = &result->doc_links.items[i];
cbm_doclink_outcome_t out = {.kind = CBM_DOCLINK_UNRESOLVED,
@@ -488,6 +513,9 @@ void cbm_doclinks_resolve_file(cbm_doclinks_t *dl, int file_idx, const CBMFileRe
mentions[nm++] = (doclink_mention_t){.src = src->id,
.tgt = out.target->id,
.line = link->line,
.target_first = out.target_first,
.target_last = out.target_last,
.via = R->via ? R->via : "doc_comment",
.syntax = link->syntax,
.exact = out.exact};
continue;
src/pipeline/doc_links.h
+48
View File
@@ -67,6 +67,10 @@ typedef struct {
const cbm_gbuf_node_t *target;
bool exact; /* tier: "exact" (qualified/doc ID/alias) vs "unique" (scope lookup) */
int reason;
/* Lines of the target that the reference names (a document's `file#L3-L9`):
* the edge's "target_lines". 0 and 0: the reference names no lines. */
uint32_t target_first;
uint32_t target_last;
} cbm_doclink_outcome_t;
/* A file's stored doc-link scope (incremental runs: files not re-extracted). */
@@ -194,6 +198,12 @@ typedef struct {
* of the file, released after its last. A NULL state (no hook, or memory
* ran out) changes the cost of resolving, never an outcome. */
void *(*file_begin)(const void *index, int run_file);
/* Optional, instead of file_begin: the file's state built from ALL its
* references before any is resolved (a PDF's cross-line joins decide
* which line fragments stand). `links` stays valid until file_end, and
* resolve's `link` points into it. */
void *(*file_prepare)(const void *index, int run_file, const CBMDocLink *links, int n,
const cbm_gbuf_t *graph);
void (*file_end)(void *state);
/* Resolve one reference of run file `run_file` (its own scope is in the
* index). Thread-safe: the index is read-only after build; `state` is the
@@ -222,9 +232,40 @@ typedef struct {
* the run. */
int (*scope_accepted)(const char *scope);
char *(*rejected_scope)(const char *scope);
/* The `via` of its MENTIONS edges: what kind of text the reference is
* written in. NULL: "doc_comment". */
const char *via;
} cbm_doclink_resolver_t;
extern const cbm_doclink_resolver_t cbm_doclink_cs_resolver;
/* Markdown documents (doc_links_md.c): via "markdown". */
extern const cbm_doclink_resolver_t cbm_doclink_md_resolver;
/* PDF documents (doc_links_pdf.c): via "pdf". */
extern const cbm_doclink_resolver_t cbm_doclink_pdf_resolver;
/* reStructuredText documents and the Python scope they read (doc_links_rst.c):
* via "rst". */
extern const cbm_doclink_resolver_t cbm_doclink_rst_resolver;
/* AsciiDoc documents and the Antora YAML scope they read (doc_links_adoc.c):
* via "asciidoc". */
extern const cbm_doclink_resolver_t cbm_doclink_adoc_resolver;
/* Over a Markdown resolver index (cbm_doclink_md_resolver.build), for the
* other document resolvers: the innermost code definition of `path` holding
* lines [first, last] (or the one named `member`), NULL for none; the Folder
* node at `path`; whether `path` is test code (a test directory or file). */
const cbm_gbuf_node_t *cbm_doclink_md_segment(const void *md_index, const char *path,
uint32_t first, uint32_t last, const char *member);
const cbm_gbuf_node_t *cbm_doclink_md_folder(const void *md_index, const char *path);
/* The definition a region [first, last] of `path` (its text in `text`) binds,
* by the field test's rule (H8 bind_range): edge blank and comment lines
* trimmed; the innermost enclosing definition, unless it is a type (or there
* is none) and the definitions inside cover every other line of the region
* (an annotation right above one counts) -- then that one definition (the
* type for several). NULL: the file. */
const cbm_gbuf_node_t *cbm_doclink_md_region(const void *md_index, const char *path,
const char *text, size_t text_len, uint32_t first,
uint32_t last);
bool cbm_doclink_md_test_path(const char *path);
#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
/* Test seam (doc_links_cs.c): the scope levels and overloads the C# resolver
@@ -248,6 +289,13 @@ bool cbm_doclink_cs_test_scope_parses(const char *scope);
void cbm_doclink_cs_test_unit_memo(bool on);
#endif
/* True for a document language whose references may bind the LINES of a file
* (Markdown `file#L3-L9`, reST :lines:, AsciiDoc tags and lines=): a body
* edit of that file keeps its
* names but can move what those lines hold, so an incremental run
* re-resolves such documents with the edited file. */
bool cbm_doclinks_binds_lines(CBMLanguage lang);
/* True when `rel_path` is a scope input of some language (scope_input). */
bool cbm_doclinks_is_scope_input(const char *rel_path);
src/pipeline/doc_links_adoc.c
+919
View File
@@ -0,0 +1,919 @@
/*
* doc_links_adoc.c — AsciiDoc references -> MENTIONS edges, via "asciidoc"
* (the resolving half of internal/cbm/doc_adoc.c), with the Antora rules the
* field test's harvester measured (H8: includes 97.6-100 % resolved, ~100 %
* precise; javadoc attributes 320/320).
*
* Antora: a component is the directory of an antora.yml; its pages live under
* <component>/modules/<module>/pages. Its attributes and collector scans, and
* a playbook's attributes, come from the YAML scope blobs (doc_adoc.c), so an
* incremental run resolves as a full one.
*
* include::target[...]: references the page left are expanded from the
* component's attributes, then the playbook's. A resource ID
* ([[version@]component:]module:]family$path) names
* <component>/modules/<module>/<family dir>/<path>, or a file a collector scan
* contributes there; any other target is relative to the including file. Its
* `lines=` bind the innermost definition holding them; its `tag=` / `tags=`
* regions -- the included file's own `tag::x[]` .. `end::x[]` comments, read
* from that indexed file -- likewise; a tag the file does not have binds the
* file (the document stays a dependent of that file, so it re-resolves when
* the file changes, on an incremental run as on a full one).
*
* {attribute} references whose value is an Antora javadoc link
* ({javadoc-root}/<module>/<package path>/<Type>.html[#member]) bind the
* Java or Kotlin type (or its member) that the repository's main sources
* declare under that package path.
*
* Monospace spans: the Markdown resolver's code-span rules (doc_links_md.c).
*/
#include "pipeline/doc_links.h"
#include "discover/discover.h" /* cbm_language_for_filename */
#include "doclink.h"
#include "foundation/arena.h"
#include "foundation/compat_fs.h"
#include "foundation/constants.h"
#include "foundation/mem_core.h"
#include "graph_buffer/graph_buffer.h"
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
enum {
RA_PATH_CAP = 1024,
RA_FIELDS = 6,
RA_EXPAND_PASSES = 10, /* nested attribute references (H8's depth) */
RA_TAG_FILE_MAX = 8 * 1024 * 1024, /* an included file read for its tags */
RA_TAGS = 16, /* tags of one include */
RA_TAG_LEN = 128, /* a tag name */
RA_INIT = 8,
};
typedef struct {
const char *key;
const char *value;
} ra_attr_t;
typedef struct {
const char *dir;
const char *into;
} ra_scan_t;
typedef struct {
const char *root; /* the antora.yml's directory ("" for the repository root) */
const char *name;
ra_attr_t *attrs;
int nattrs;
ra_scan_t *scans;
int nscans;
} ra_comp_t;
typedef struct {
const char *project;
const char *repo; /* the repository on disk: included files are read for their tags */
void *md;
const char **run_paths;
int run_count;
CBMArena arena;
ra_comp_t *comps;
int ncomps;
ra_attr_t *pattrs; /* playbook attributes, the first definition of a key wins */
int npattrs;
} ra_index_t;
static void ra_destroy(void *index);
static const char *ra_base(const char *path) {
const char *s = strrchr(path, '/');
return s ? s + SKIP_ONE : path;
}
static bool ra_ends(const char *s, const char *suffix) {
size_t n = strlen(s);
size_t k = strlen(suffix);
return n >= k && strcmp(s + n - k, suffix) == 0;
}
static bool ra_grow(CBMArena *a, void **items, int count, int *cap, size_t size) {
if (count < *cap) {
return true;
}
int ncap = *cap ? *cap * PAIR_LEN : RA_INIT;
void *grown = cbm_arena_alloc(a, (size_t)ncap * size);
if (!grown) {
return false;
}
if (count > 0) {
memcpy(grown, *items, (size_t)count * size);
}
*items = grown;
*cap = ncap;
return true;
}
/* One YAML scope blob: a component (C) or a playbook (P). */
static bool ra_read_scope(ra_index_t *x, const char *path, const char *scope, int *comp_cap,
int *pattr_cap) {
char *copy = cbm_arena_strdup(&x->arena, scope);
if (!copy) {
return false;
}
char *save = NULL;
(void)strtok_r(copy, "\n", &save); /* the tag line */
char *line = strtok_r(NULL, "\n", &save);
if (!line || (strcmp(line, "C") != 0 && strcmp(line, "P") != 0)) {
return true;
}
bool component = line[0] == 'C';
ra_comp_t comp = {0};
int acap = 0;
int scap = 0;
if (component) {
const char *b = ra_base(path);
comp.root =
cbm_arena_strndup(&x->arena, path, b > path ? (size_t)(b - path - SKIP_ONE) : 0);
comp.name = "";
if (!comp.root) {
return false;
}
}
for (line = strtok_r(NULL, "\n", &save); line; line = strtok_r(NULL, "\n", &save)) {
char *tab = strchr(line, '\t');
if (!tab) {
continue;
}
*tab = '\0';
char *v = tab + SKIP_ONE;
if (strcmp(line, "N") == 0 && component) {
comp.name = v;
} else if (strcmp(line, "A") == 0) {
char *t2 = strchr(v, '\t');
if (!t2) {
continue;
}
*t2 = '\0';
ra_attr_t at = {v, t2 + SKIP_ONE};
if (component) {
if (!ra_grow(&x->arena, (void **)&comp.attrs, comp.nattrs, &acap, sizeof(at))) {
return false;
}
comp.attrs[comp.nattrs++] = at;
} else {
bool seen = false;
for (int i = 0; i < x->npattrs; i++) {
seen = seen || strcmp(x->pattrs[i].key, at.key) == 0;
}
if (!seen) {
if (!ra_grow(&x->arena, (void **)&x->pattrs, x->npattrs, pattr_cap,
sizeof(at))) {
return false;
}
x->pattrs[x->npattrs++] = at;
}
}
} else if (strcmp(line, "D") == 0 && component) {
if (!ra_grow(&x->arena, (void **)&comp.scans, comp.nscans, &scap, sizeof(ra_scan_t))) {
return false;
}
comp.scans[comp.nscans++] = (ra_scan_t){v, ""};
} else if (strcmp(line, "I") == 0 && component && comp.nscans > 0) {
comp.scans[comp.nscans - 1].into = v;
}
}
if (component) {
if (!ra_grow(&x->arena, (void **)&x->comps, x->ncomps, comp_cap, sizeof(comp))) {
return false;
}
x->comps[x->ncomps++] = comp;
}
return true;
}
static void *ra_build(const cbm_doclink_build_in_t *in) {
ra_index_t *x = (ra_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*x));
if (!x) {
return NULL;
}
cbm_arena_init(&x->arena);
x->project = in->ctx ? in->ctx->project_name : NULL;
x->repo = in->ctx ? in->ctx->repo_path : NULL;
x->run_count = in->run_file_count;
x->run_paths = in->run_file_count > 0
? (const char **)cbm_calloc(CBM_MEM_CLASS_OTHER,
(size_t)in->run_file_count * sizeof(char *))
: NULL;
x->md = cbm_doclink_md_resolver.build(in);
if ((in->run_file_count > 0 && !x->run_paths) || !x->md || !x->project) {
ra_destroy(x);
return NULL;
}
int comp_cap = 0;
int pattr_cap = 0;
for (int i = 0; i < in->file_count; i++) {
const cbm_doclink_file_t *f = &in->files[i];
if (f->run_file >= 0 && f->run_file < x->run_count) {
x->run_paths[f->run_file] = f->rel_path; /* valid until destroy */
}
if (f->scope &&
strncmp(f->scope, CBM_DOCLINK_ADOC_SCOPE_TAG "\n",
strlen(CBM_DOCLINK_ADOC_SCOPE_TAG "\n")) == 0 &&
!ra_read_scope(x, f->rel_path, f->scope, &comp_cap, &pattr_cap)) {
ra_destroy(x);
return NULL;
}
}
return x;
}
static void ra_destroy(void *index) {
ra_index_t *x = (ra_index_t *)index;
if (!x) {
return;
}
if (x->md) {
cbm_doclink_md_resolver.destroy(x->md);
}
cbm_arena_destroy(&x->arena);
cbm_free(CBM_MEM_CLASS_OTHER, x->run_paths);
cbm_free(CBM_MEM_CLASS_OTHER, x);
}
/* H8 component_of: the deepest component whose modules/ hold the document. */
static const ra_comp_t *ra_component_of(const ra_index_t *x, const char *doc) {
const ra_comp_t *best = NULL;
for (int i = 0; i < x->ncomps; i++) {
const ra_comp_t *c = &x->comps[i];
char pref[RA_PATH_CAP];
snprintf(pref, sizeof(pref), "%s%smodules/", c->root, c->root[0] ? "/" : "");
if (strncmp(doc, pref, strlen(pref)) == 0 &&
(!best || strlen(c->root) > strlen(best->root))) {
best = c;
}
}
return best;
}
static const char *ra_attr(const ra_index_t *x, const ra_comp_t *c, const char *key, size_t n) {
for (int i = 0; c && i < c->nattrs; i++) {
if (strlen(c->attrs[i].key) == n && memcmp(c->attrs[i].key, key, n) == 0) {
return c->attrs[i].value;
}
}
for (int i = 0; i < x->npattrs; i++) {
if (strlen(x->pattrs[i].key) == n && memcmp(x->pattrs[i].key, key, n) == 0) {
return x->pattrs[i].value;
}
}
return NULL;
}
static bool ra_word(char c) {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
}
/* The text with the component's and the playbook's attribute references
* substituted, pass after pass. False when one stays undefined or it does
* not fit. */
static bool ra_expand(const ra_index_t *x, const ra_comp_t *c, const char *s, char *out,
size_t cap) {
char cur[RA_PATH_CAP];
char nxt[RA_PATH_CAP];
if (snprintf(cur, sizeof(cur), "%s", s) >= (int)sizeof(cur)) {
return false;
}
bool undefined = false;
for (int pass = 0; pass < RA_EXPAND_PASSES; pass++) {
size_t w = 0;
bool changed = false;
undefined = false;
for (size_t i = 0; cur[i]; i++) {
const char *v = NULL;
size_t e = i + SKIP_ONE;
if (cur[i] == '{' && (i == 0 || cur[i - SKIP_ONE] != '\\') && ra_word(cur[e])) {
while (cur[e] && (ra_word(cur[e]) || cur[e] == '-')) {
e++;
}
if (cur[e] == '}') {
v = ra_attr(x, c, cur + i + SKIP_ONE, e - i - SKIP_ONE);
undefined = undefined || !v;
}
}
if (v) {
size_t vl = strlen(v);
if (w + vl >= sizeof(nxt)) {
return false;
}
memcpy(nxt + w, v, vl);
w += vl;
i = e;
changed = true;
continue;
}
if (w + SKIP_ONE >= sizeof(nxt)) {
return false;
}
nxt[w++] = cur[i];
}
nxt[w] = '\0';
memcpy(cur, nxt, w + SKIP_ONE);
if (!changed) {
break;
}
}
if (undefined || snprintf(out, cap, "%s", cur) >= (int)cap) {
return false;
}
return true;
}
/* os.path.normpath(base/p); false when it leaves the repository. */
static bool ra_norm(const char *base, const char *p, char *out, size_t cap) {
char tmp[RA_PATH_CAP * PAIR_LEN];
snprintf(tmp, sizeof(tmp), "%s%s%s", base, base[0] && p[0] ? "/" : "", p);
size_t w = 0;
out[0] = '\0';
for (char *s = tmp; *s;) {
char *e = strchr(s, '/');
size_t l = e ? (size_t)(e - s) : strlen(s);
if (l == PAIR_LEN && s[0] == '.' && s[1] == '.') {
if (w == 0) {
return false;
}
while (w > 0 && out[w - SKIP_ONE] != '/') {
w--;
}
if (w > 0) {
w--;
}
out[w] = '\0';
} else if (l > 0 && !(l == SKIP_ONE && s[0] == '.')) {
if (w + l + PAIR_LEN > cap) {
return false;
}
if (w > 0) {
out[w++] = '/';
}
memcpy(out + w, s, l);
w += l;
out[w] = '\0';
}
if (!e) {
break;
}
s = e + SKIP_ONE;
}
return true;
}
static void ra_unresolved(cbm_doclink_outcome_t *out, int reason) {
out->kind = CBM_DOCLINK_UNRESOLVED;
out->reason = reason;
}
static void ra_edge(cbm_doclink_outcome_t *out, const cbm_gbuf_node_t *n, uint32_t a, uint32_t b) {
out->kind = CBM_DOCLINK_EDGE;
out->target = n;
out->exact = true;
out->target_first = a;
out->target_last = b;
}
/* A path of a type the indexer does not read (a service file, an image): no
* reference to code, decided by its name as the Markdown resolver does -- never
* by the disk, which an incremental run does not watch for such files. */
static bool ra_unindexable(const char *path) {
return cbm_language_for_filename(ra_base(path)) == CBM_LANG_COUNT;
}
/* A collector scan of build output (`./build/...`): what it contributes is
* generated, not a file of the repository (H8). */
static bool ra_build_dir(const char *dir) {
while (dir[0] == '.' && dir[1] == '/') {
dir += PAIR_LEN;
}
return strcmp(dir, "build") == 0 || strncmp(dir, "build/", strlen("build/")) == 0 ||
strstr(dir, "/build/") != NULL;
}
static const char *const RA_FAMILIES[][2] = {{"example", "examples"},
{"partial", "partials"},
{"page", "pages"},
{"image", "images"},
{"attachment", "attachments"}};
/* H8 resolve_target: an Antora resource ID or a path relative to the including
* file. Returns the repository path in `out`, or false with `out->kind` set. */
static bool ra_target(const ra_index_t *x, const cbm_gbuf_t *graph, const char *doc,
const ra_comp_t *comp, const char *target, char *path, size_t cap,
cbm_doclink_outcome_t *out) {
out->kind = CBM_DOCLINK_LOCAL;
if (strstr(target, "://")) {
return false; /* a URL include */
}
const char *dollar = strchr(target, '$');
if (dollar && comp) {
char pre[RA_PATH_CAP];
snprintf(pre, sizeof(pre), "%.*s", (int)(dollar - target), target);
const char *at = strrchr(pre, '@');
char *segs = at ? pre + (at - pre) + SKIP_ONE : pre;
/* family is the last segment; module and component before it */
char *fam = strrchr(segs, ':');
const char *module = NULL;
if (fam) {
*fam = '\0';
fam++;
char *comp_sep = strchr(segs, ':');
if (comp_sep) {
*comp_sep = '\0';
if (strcmp(segs, comp->name) != 0) {
return false; /* another component's resource */
}
module = comp_sep + SKIP_ONE;
} else {
module = segs;
}
} else {
fam = segs;
}
char doc_module[RA_PATH_CAP] = "ROOT";
if (!module) {
/* the including page's own module: modules/<module>/... */
const char *rel = doc + strlen(comp->root) + (comp->root[0] ? SKIP_ONE : 0);
if (strncmp(rel, "modules/", strlen("modules/")) == 0) {
const char *m = rel + strlen("modules/");
const char *slash = strchr(m, '/');
if (slash) {
snprintf(doc_module, sizeof(doc_module), "%.*s", (int)(slash - m), m);
}
}
module = doc_module;
}
const char *famdir = NULL;
for (size_t i = 0; i < sizeof(RA_FAMILIES) / sizeof(RA_FAMILIES[0]); i++) {
if (strcmp(fam, RA_FAMILIES[i][0]) == 0) {
famdir = RA_FAMILIES[i][1];
}
}
if (!famdir) {
ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE);
return false;
}
char virt[RA_PATH_CAP];
snprintf(virt, sizeof(virt), "modules/%s/%s/%s", module, famdir, dollar + SKIP_ONE);
if (ra_norm(comp->root, virt, path, cap) &&
cbm_pipeline_file_node(graph, x->project, path)) {
return true;
}
/* a file a collector scan contributes into the component */
for (int i = 0; i < comp->nscans; i++) {
const char *into = comp->scans[i].into;
while (into[0] == '.' && into[1] == '/') {
into += PAIR_LEN;
}
size_t il = strlen(into);
while (il > 0 && into[il - SKIP_ONE] == '/') {
il--;
}
if (il == 0 || strncmp(virt, into, il) != 0 || virt[il] != '/') {
continue;
}
char sub[RA_PATH_CAP];
snprintf(sub, sizeof(sub), "%s/%s", comp->scans[i].dir, virt + il + SKIP_ONE);
if (ra_norm(comp->root, sub, path, cap) &&
cbm_pipeline_file_node(graph, x->project, path)) {
return true;
}
if (ra_build_dir(comp->scans[i].dir)) {
return false; /* build output (H8 generated): no file of the repository */
}
}
if (ra_unindexable(virt)) {
return false;
}
ra_unresolved(out, CBM_DOCLINK_REASON_MISSING);
return false;
}
char dir[RA_PATH_CAP];
const char *b = ra_base(doc);
snprintf(dir, sizeof(dir), "%.*s", b > doc ? (int)(b - doc - SKIP_ONE) : 0, doc);
if (!ra_norm(dir, target, path, cap)) {
return false; /* above the repository root */
}
if (cbm_pipeline_file_node(graph, x->project, path)) {
return true;
}
char parent[RA_PATH_CAP];
const char *pb = ra_base(path);
snprintf(parent, sizeof(parent), "%.*s", pb > path ? (int)(pb - path - SKIP_ONE) : 0, path);
if (!ra_unindexable(path) && parent[0] && cbm_doclink_md_folder(x->md, parent)) {
ra_unresolved(out, CBM_DOCLINK_REASON_MISSING);
}
return false;
}
/* Asciidoctor's TagDirectiveRx \b(?:tag|(e)nd)::(\S+?)\[\](?=$|[ \r]) over the
* file: the 1-based lines from the first region's first content line to the
* last region's last one (directive lines excluded). False when a tag has no
* region. */
static bool ra_tag_span(const char *src, size_t n, const char *const *tags, int ntags,
uint32_t *first, uint32_t *last) {
*first = 0;
*last = 0;
for (int t = 0; t < ntags; t++) {
size_t tl = strlen(tags[t]);
uint32_t line = 1;
uint32_t start = 0;
bool found = false;
size_t pos = 0;
while (pos < n) {
const char *nl = memchr(src + pos, '\n', n - pos);
size_t end = nl ? (size_t)(nl - src) : n;
for (size_t i = pos; i + PAIR_LEN < end; i++) {
bool is_end = false;
size_t k;
if (i + strlen("tag::") <= end && memcmp(src + i, "tag::", strlen("tag::")) == 0) {
k = i + strlen("tag::");
} else if (i + strlen("end::") <= end &&
memcmp(src + i, "end::", strlen("end::")) == 0) {
k = i + strlen("end::");
is_end = true;
} else {
continue;
}
if (i > pos && ra_word(src[i - SKIP_ONE])) {
continue; /* \b */
}
if (k + tl + PAIR_LEN > end || memcmp(src + k, tags[t], tl) != 0 ||
src[k + tl] != '[' || src[k + tl + SKIP_ONE] != ']') {
continue;
}
size_t after = k + tl + PAIR_LEN;
if (after != end && src[after] != ' ' && src[after] != '\r') {
continue;
}
if (!is_end && start == 0) {
start = line + SKIP_ONE;
} else if (is_end && start != 0) {
if (start <= line - SKIP_ONE) {
*first = (*first == 0 || start < *first) ? start : *first;
*last = line - SKIP_ONE > *last ? line - SKIP_ONE : *last;
found = true;
}
start = 0;
}
break;
}
line++;
pos = end + SKIP_ONE;
}
if (start != 0 && start <= line - SKIP_ONE) {
*first = (*first == 0 || start < *first) ? start : *first;
*last = line - SKIP_ONE > *last ? line - SKIP_ONE : *last;
found = true;
}
if (!found) {
return false;
}
}
return *first > 0;
}
/* The included file's text, read from the indexed file on disk (at most
* RA_TAG_FILE_MAX bytes): its tag regions and the lines of a region. NULL
* when it cannot be read; the caller frees it (cbm_free, OTHER). */
static char *ra_read(const ra_index_t *x, const char *path, size_t *n) {
*n = 0;
if (!x->repo) {
return NULL;
}
char abs[RA_PATH_CAP * PAIR_LEN];
snprintf(abs, sizeof(abs), "%s/%s", x->repo, path);
FILE *f = cbm_fopen(abs, "rb");
if (!f) {
return NULL;
}
char *buf = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, RA_TAG_FILE_MAX);
*n = buf ? fread(buf, SKIP_ONE, RA_TAG_FILE_MAX, f) : 0;
(void)fclose(f);
return buf;
}
/* `lines=a..b;c;d..-1`: the span of all ranges (-1: the file's end). */
static bool ra_lines(const char *v, uint32_t *first, uint32_t *last) {
*first = 0;
*last = 0;
const char *p = v;
while (*p) {
while (*p == ';' || *p == ',' || *p == ' ' || *p == '"') {
p++;
}
if (!*p) {
break;
}
if (*p < '0' || *p > '9') {
return false;
}
uint32_t a = (uint32_t)strtoul(p, (char **)&p, 10);
uint32_t b = a;
if (p[0] == '.' && p[1] == '.') {
p += PAIR_LEN;
if (p[0] == '-' && p[1] == '1') {
b = UINT32_MAX;
p += PAIR_LEN;
} else if (*p >= '0' && *p <= '9') {
b = (uint32_t)strtoul(p, (char **)&p, 10);
} else {
return false;
}
}
if (a == 0 || b < a) {
return false;
}
*first = (*first == 0 || a < *first) ? a : *first;
*last = b > *last ? b : *last;
}
return *first > 0;
}
typedef struct {
char buf[RA_PATH_CAP * PAIR_LEN];
char *f[RA_FIELDS];
int n;
} ra_rec_t;
static bool ra_record(const char *raw, ra_rec_t *r) {
if (snprintf(r->buf, sizeof(r->buf), "%s", raw) >= (int)sizeof(r->buf)) {
return false;
}
r->n = 0;
char *p = r->buf;
while (r->n < RA_FIELDS) {
r->f[r->n++] = p;
char *tab = strchr(p, '\t');
if (!tab) {
break;
}
*tab = '\0';
p = tab + SKIP_ONE;
}
for (int i = r->n; i < RA_FIELDS; i++) {
r->f[i] = "";
}
return true;
}
static void ra_include(const ra_index_t *x, const cbm_gbuf_t *graph, const char *doc,
const ra_rec_t *r, cbm_doclink_outcome_t *out) {
const ra_comp_t *comp = ra_component_of(x, doc);
char target[RA_PATH_CAP];
if (strchr(r->f[1], '{')) {
if (!comp || !ra_expand(x, comp, r->f[1], target, sizeof(target))) {
ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE); /* an undefined attribute */
return;
}
} else {
snprintf(target, sizeof(target), "%s", r->f[1]);
}
char path[RA_PATH_CAP];
if (!ra_target(x, graph, doc, comp, target, path, sizeof(path), out)) {
return;
}
const cbm_gbuf_node_t *file = cbm_pipeline_file_node(graph, x->project, path);
if (!file) {
out->kind = CBM_DOCLINK_LOCAL;
return;
}
uint32_t first = 0;
uint32_t last = 0;
const char *tagv = r->f[2];
if (!tagv[0] && r->f[3][0] && !ra_lines(r->f[3], &first, &last)) {
ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE);
return;
}
size_t tn = 0;
char *text = (tagv[0] || first > 0) ? ra_read(x, path, &tn) : NULL;
if (tagv[0]) {
/* H8: negated and wildcard tags leave the positive ones; none left
* includes the whole file */
char tags[RA_TAGS][RA_TAG_LEN];
const char *tagp[RA_TAGS];
int nt = 0;
for (const char *p = tagv; *p && nt < RA_TAGS;) {
size_t l = strcspn(p, ";,");
if (l > 0 && l < RA_TAG_LEN && p[0] != '!' && !memchr(p, '*', l)) {
snprintf(tags[nt], RA_TAG_LEN, "%.*s", (int)l, p);
tagp[nt] = tags[nt];
nt++;
}
p += l + (p[l] ? SKIP_ONE : 0);
}
if (nt == 0 || !text || !ra_tag_span(text, tn, tagp, nt, &first, &last)) {
first = 0; /* wildcards only, or a tag the file lacks: the file */
last = 0;
}
}
if (first > 0) {
const cbm_gbuf_node_t *seg = text
? cbm_doclink_md_region(x->md, path, text, tn, first, last)
: cbm_doclink_md_segment(x->md, path, first, last, NULL);
cbm_free(CBM_MEM_CLASS_OTHER, text);
ra_edge(out, seg ? seg : file, first, last == UINT32_MAX ? 0 : last);
return;
}
cbm_free(CBM_MEM_CLASS_OTHER, text);
ra_edge(out, file, 0, 0);
}
static bool ra_class_label(const char *label) {
static const char *const labels[] = {"Class", "Interface", "Enum", "Type", "Trait", NULL};
for (int i = 0; label && labels[i]; i++) {
if (strcmp(label, labels[i]) == 0) {
return true;
}
}
return false;
}
static bool ra_member_label(const char *label) {
static const char *const labels[] = {"Method", "Field", "Variable", "Function", NULL};
for (int i = 0; label && labels[i]; i++) {
if (strcmp(label, labels[i]) == 0) {
return true;
}
}
return false;
}
/* H8 javadoc: xref:attachment$api/<module>/<pkg path>/<Type>.html[#member]. */
static void ra_javadoc(const ra_index_t *x, const cbm_gbuf_t *graph, const char *val,
cbm_doclink_outcome_t *out) {
static const char key[] = "xref:attachment$api/";
const char *p = strstr(val, key);
if (!p) {
return;
}
p += strlen(key);
const char *slash = strchr(p, '/'); /* after the JPMS module */
if (!slash) {
return;
}
const char *path = slash + SKIP_ONE;
const char *html = strstr(path, ".html");
if (!html || memchr(path, '[', (size_t)(html - path))) {
return;
}
char rel[RA_PATH_CAP];
snprintf(rel, sizeof(rel), "%.*s", (int)(html - path), path);
const char *tname = ra_base(rel);
if (strcmp(tname, "package-summary") == 0 || strcmp(tname, "module-summary") == 0 ||
strcmp(rel, "index") == 0) {
return; /* a package or module page: no code node */
}
char anchor[RA_PATH_CAP] = "";
if (html[strlen(".html")] == '#') {
const char *a = html + strlen(".html") + SKIP_ONE;
size_t al = strcspn(a, "[(-");
snprintf(anchor, sizeof(anchor), "%.*s", (int)al, a);
}
char pkgdir[RA_PATH_CAP];
snprintf(pkgdir, sizeof(pkgdir), "%.*s", (int)(tname - rel), rel); /* "org/x/" */
char outer[RA_PATH_CAP];
snprintf(outer, sizeof(outer), "%.*s", (int)strcspn(tname, "."), tname);
const char *want = strrchr(tname, '.') ? strrchr(tname, '.') + SKIP_ONE : tname;
/* the source files declaring <pkg dir>/<Outer>.java|.kt, main sources first */
const cbm_gbuf_node_t **nodes = NULL;
int count = 0;
if (cbm_gbuf_find_by_name(graph, outer, &nodes, &count) != 0) {
return;
}
const char *files[2] = {NULL, NULL};
int nfiles = 0;
for (int pass = 0; pass < PAIR_LEN && nfiles == 0; pass++) {
bool main_only = pass == 0;
for (int i = 0; i < count; i++) {
const cbm_gbuf_node_t *n = nodes[i];
const char *fp = n->file_path;
if (!ra_class_label(n->label) || !fp) {
continue;
}
char tail[RA_PATH_CAP * PAIR_LEN];
bool java = ra_ends(fp, ".java");
snprintf(tail, sizeof(tail), "%s%s%s", pkgdir, outer, java ? ".java" : ".kt");
size_t fl = strlen(fp);
size_t tl = strlen(tail);
if (!(java || ra_ends(fp, ".kt")) || fl < tl || strcmp(fp + fl - tl, tail) != 0 ||
(fl > tl && fp[fl - tl - SKIP_ONE] != '/') ||
(main_only && !strstr(fp, "/src/main/"))) {
continue;
}
bool seen = false;
for (int k = 0; k < nfiles; k++) {
seen = seen || strcmp(files[k], fp) == 0;
}
if (!seen && nfiles < PAIR_LEN) {
files[nfiles++] = fp;
} else if (!seen) {
nfiles++;
}
}
}
if (nfiles == 0) {
ra_unresolved(out, CBM_DOCLINK_REASON_MISSING);
return;
}
if (nfiles > SKIP_ONE) {
ra_unresolved(out, CBM_DOCLINK_REASON_AMBIGUOUS);
return;
}
/* the type in that file (the first by line for a repeated name) */
const cbm_gbuf_node_t *type = NULL;
if (cbm_gbuf_find_by_name(graph, want, &nodes, &count) == 0) {
for (int i = 0; i < count; i++) {
const cbm_gbuf_node_t *n = nodes[i];
if (ra_class_label(n->label) && n->file_path && strcmp(n->file_path, files[0]) == 0 &&
(!type || n->start_line < type->start_line ||
(n->start_line == type->start_line && n->id < type->id))) {
type = n;
}
}
}
if (!type) {
ra_unresolved(out, CBM_DOCLINK_REASON_GRAPH_GAP);
return;
}
const cbm_gbuf_node_t *member = NULL;
int members = 0;
if (anchor[0] && cbm_gbuf_find_by_name(graph, anchor, &nodes, &count) == 0) {
for (int i = 0; i < count; i++) {
const cbm_gbuf_node_t *n = nodes[i];
if (ra_member_label(n->label) && n->file_path && strcmp(n->file_path, files[0]) == 0 &&
n->start_line >= type->start_line && n->start_line <= type->end_line) {
member = n;
members++;
}
}
}
/* an overloaded member: the type that declares it */
ra_edge(out, members == SKIP_ONE ? member : type, 0, 0);
}
static void ra_attribute(const ra_index_t *x, const cbm_gbuf_t *graph, const char *doc,
const ra_rec_t *r, cbm_doclink_outcome_t *out) {
const ra_comp_t *comp = ra_component_of(x, doc);
if (!comp) {
return; /* outside an Antora component: no API attributes (H8) */
}
const char *raw = r->f[2][0] ? r->f[2] : ra_attr(x, comp, r->f[1], strlen(r->f[1]));
if (!raw || !strstr(raw, "{javadoc-root}")) {
return;
}
char val[RA_PATH_CAP];
if (!ra_expand(x, comp, r->f[2][0] ? r->f[3] : raw, val, sizeof(val))) {
return;
}
ra_javadoc(x, graph, val, out);
}
static void ra_resolve(const void *index, void *state, int run_file, const CBMDocLink *link,
const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) {
(void)state;
const ra_index_t *x = (const ra_index_t *)index;
out->kind = CBM_DOCLINK_LOCAL;
const char *doc = (run_file >= 0 && run_file < x->run_count) ? x->run_paths[run_file] : NULL;
if (!doc || !link->raw) {
ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE);
return;
}
if (link->syntax == CBM_DOCLINK_ADOC_CODE_PATH || link->syntax == CBM_DOCLINK_ADOC_CODE_NAME) {
CBMDocLink md = *link;
md.syntax = link->syntax == CBM_DOCLINK_ADOC_CODE_PATH ? CBM_DOCLINK_MD_CODE_PATH
: CBM_DOCLINK_MD_CODE_NAME;
cbm_doclink_md_resolver.resolve(x->md, NULL, run_file, &md, graph, out);
return;
}
ra_rec_t *r = (ra_rec_t *)cbm_alloc(CBM_MEM_CLASS_OTHER, sizeof(*r));
if (!r || !ra_record(link->raw, r)) {
ra_unresolved(out, CBM_DOCLINK_REASON_UNPARSEABLE);
cbm_free(CBM_MEM_CLASS_OTHER, r);
return;
}
if (link->syntax == CBM_DOCLINK_ADOC_INCLUDE) {
ra_include(x, graph, doc, r, out);
} else if (link->syntax == CBM_DOCLINK_ADOC_ATTRIBUTE) {
ra_attribute(x, graph, doc, r, out);
}
cbm_free(CBM_MEM_CLASS_OTHER, r);
}
const cbm_doclink_resolver_t cbm_doclink_adoc_resolver = {
.langs = {CBM_LANG_ASCIIDOC, CBM_LANG_YAML},
.lang_count = 2,
.scope_tag = CBM_DOCLINK_ADOC_SCOPE_TAG,
.build = ra_build,
.destroy = ra_destroy,
.resolve = ra_resolve,
.via = "asciidoc",
};
src/pipeline/doc_links_md.c
+1596
View File
File diff suppressed because it is too large Load Diff
src/pipeline/doc_links_pdf.c
+811
View File
@@ -0,0 +1,811 @@
/*
* doc_links_pdf.c — PDF mentions -> MENTIONS edges: the resolving half.
*
* A PDF page (doc_pdf.c) names code structurally: a path or a qualified name.
* The tiers are the field-tested ones (E6, frozen resolver), EXACT or UNIQUE
* only:
* path-exact the mention is a File/Folder node's repository path
* path-suffix exactly one File/Folder whose path ends with the mention at a
* '/' boundary
* qn-exact exactly one node whose qualified name (project prefix off)
* is the mention
* qn-suffix exactly one node whose qualified name ends with the mention
* at a component boundary -- or whose owner-qualified name
* does (parent_class + name: a Go method's QN has no receiver)
* Two candidates that are one entity collapse first (a class and its
* constructors; duplicate nodes of one definition). A cross-line join is
* resolved before anything else in its file: when it resolves EXACT or
* UNIQUE it replaces the line fragments it was built from.
*
* Hygiene (the field test's R2 and R4): a reference that resolves to a data
* file (.json, .yaml, ...) is no code reference; one that resolves into a
* test, mock or fixture file is a test_only_target row.
*
* Unresolved: ambiguous (several entities); missing (nothing, but the
* mention's first component is a name or directory of this repository).
* Anything else names code outside the repository: LOCAL, no row.
*
* Targets exclude the labels the incremental route does not proxy (Macro,
* Comment, Section, Branch, Commit, Tag) and the field test's non-code labels
* (Project, Route, Channel, Decorator), so a re-resolved document binds what a
* full build binds.
*/
#include "pipeline/doc_links.h"
#include "doclink.h"
#include "foundation/arena.h"
#include "foundation/constants.h"
#include "foundation/hash_table.h"
#include "foundation/mem_core.h"
#include "graph_buffer/graph_buffer.h"
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
enum {
PDFR_TABLE_INIT = 1024,
PDFR_KEY_CAP = 1024, /* the longest mention form looked up */
PDFR_MAX_COMPS = 64,
PDFR_MAX_FORMS = 2,
PDFR_MEMO_INIT = 64,
};
static const char *const PDFR_EXCLUDED_LABELS[] = {
"Section", "Project", "Branch", "Route", "Channel", "Decorator", "Macro",
"Comment", "Commit", "Tag", "File", "Folder", NULL};
static const char *const PDFR_CONTAINERS[] = {"Class", "Struct", "Interface", "Enum",
"Trait", "Type", "Record", "Object",
"Protocol", "Union", NULL};
static const char *const PDFR_DATA_EXT[] = {".json", ".yaml", ".yml", ".toml",
".csv", ".tsv", ".lock", NULL};
/* R4: the frozen test-file rule, with the directories the field test found it
* missing (Foundry *.t.sol, test/, mocks/, testutil/, testing/, testdata/). */
static const char *const PDFR_TEST_DIRS[] = {"tests", "__tests__", "test", "mocks",
"testutil", "testing", "testdata", NULL};
static bool in_list(const char *s, const char *const *list) {
for (int i = 0; list[i]; i++) {
if (strcmp(s, list[i]) == 0) {
return true;
}
}
return false;
}
static bool ends_with(const char *s, const char *suf) {
size_t n = strlen(s);
size_t m = strlen(suf);
return n >= m && memcmp(s + n - m, suf, m) == 0;
}
static bool starts_with(const char *s, const char *pre) {
return strncmp(s, pre, strlen(pre)) == 0;
}
/* ── Index ───────────────────────────────────────────────────────── */
typedef struct pdfr_list {
const cbm_gbuf_node_t *node;
struct pdfr_list *next;
} pdfr_list_t;
typedef struct {
const char *project;
size_t project_len;
CBMArena arena;
CBMHashTable *paths; /* File/Folder repository path -> pdfr_list_t* */
CBMHashTable *basenames; /* last path component -> pdfr_list_t* */
CBMHashTable *dirs; /* every directory component of an indexed path (value 1) */
/* Code nodes whose qualified name does not end with their name (a Module
* is named by its file path), keyed by the QN's last component: the
* field test matched qualified names by their components, not by name. */
CBMHashTable *irregular;
bool failed;
} pdfr_index_t;
static void pdfr_destroy(void *index);
static bool list_add(pdfr_index_t *x, CBMHashTable *t, const char *key, const cbm_gbuf_node_t *n) {
pdfr_list_t *e = (pdfr_list_t *)cbm_arena_alloc(&x->arena, sizeof(*e));
if (!e) {
return false;
}
e->node = n;
e->next = (pdfr_list_t *)cbm_ht_get(t, key);
cbm_ht_set(t, key, e);
return true;
}
static bool index_paths(pdfr_index_t *x, const cbm_gbuf_t *graph, const char *label) {
const cbm_gbuf_node_t **nodes = NULL;
int count = 0;
if (cbm_gbuf_find_by_label(graph, label, &nodes, &count) != 0) {
return true;
}
for (int i = 0; i < count; i++) {
const char *fp = nodes[i]->file_path;
if (!fp || !fp[0] || fp[0] == '<') {
continue;
}
while (*fp == '/') {
fp++;
}
size_t n = strlen(fp);
while (n > 0 && fp[n - 1] == '/') {
n--;
}
char *p = cbm_arena_strndup(&x->arena, fp, n);
if (!p || !list_add(x, x->paths, p, nodes[i])) {
return false;
}
const char *slash = strrchr(p, '/');
if (!list_add(x, x->basenames, slash ? slash + 1 : p, nodes[i])) {
return false;
}
/* directory components, for "is this a name of the repository" */
const char *s = p;
for (const char *q = p;; q++) {
if (*q == '/' || *q == '\0') {
if (q > s) {
char *c = cbm_arena_strndup(&x->arena, s, (size_t)(q - s));
if (!c) {
return false;
}
cbm_ht_set(x->dirs, c, (void *)(uintptr_t)1);
}
if (!*q) {
break;
}
s = q + 1;
}
}
}
return true;
}
static bool code_node(const pdfr_index_t *x, const cbm_gbuf_node_t *n);
static const char *rel_qn(const pdfr_index_t *x, const char *qn);
static void index_irregular(const cbm_gbuf_node_t *n, void *ud) {
pdfr_index_t *x = (pdfr_index_t *)ud;
if (x->failed || !code_node(x, n)) {
return;
}
const char *rel = rel_qn(x, n->qualified_name);
const char *dot = strrchr(rel, '.');
const char *last = dot ? dot + 1 : rel;
if (n->name && strcmp(last, n->name) == 0) {
return;
}
if (!list_add(x, x->irregular, last, n)) {
x->failed = true;
}
}
static void *pdfr_build(const cbm_doclink_build_in_t *in) {
if (in->file_count == 0) {
return NULL; /* no PDF in the repository: nothing to resolve */
}
pdfr_index_t *x = (pdfr_index_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*x));
if (!x) {
return NULL;
}
cbm_arena_init(&x->arena);
x->project = in->ctx ? in->ctx->project_name : NULL;
x->project_len = x->project ? strlen(x->project) : 0;
x->paths = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, PDFR_TABLE_INIT);
x->basenames = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, PDFR_TABLE_INIT);
x->dirs = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, PDFR_TABLE_INIT);
x->irregular = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, PDFR_TABLE_INIT);
if (!x->project || !x->paths || !x->basenames || !x->dirs || !x->irregular ||
!index_paths(x, in->graph, "File") || !index_paths(x, in->graph, "Folder")) {
pdfr_destroy(x);
return NULL;
}
cbm_gbuf_foreach_node(in->graph, index_irregular, x);
if (x->failed) {
pdfr_destroy(x);
return NULL;
}
return x;
}
static void pdfr_destroy(void *index) {
pdfr_index_t *x = (pdfr_index_t *)index;
if (!x) {
return;
}
cbm_ht_free(x->paths);
cbm_ht_free(x->basenames);
cbm_ht_free(x->dirs);
cbm_ht_free(x->irregular);
cbm_arena_destroy(&x->arena);
cbm_free(CBM_MEM_CLASS_OTHER, x);
}
/* ── Candidate sets ──────────────────────────────────────────────── */
typedef struct {
const cbm_gbuf_node_t **v;
int n;
int cap;
bool fail;
} nodeset_t;
static void ns_add(nodeset_t *s, const cbm_gbuf_node_t *n) {
for (int i = 0; i < s->n; i++) {
if (s->v[i] == n) {
return;
}
}
if (s->fail) {
return;
}
if (s->n == s->cap) {
int ncap = s->cap ? s->cap * 2 : 8;
const cbm_gbuf_node_t **g = (const cbm_gbuf_node_t **)cbm_realloc(
CBM_MEM_CLASS_OTHER, (void *)s->v, (size_t)ncap * sizeof(*g));
if (!g) {
s->fail = true;
return;
}
s->v = g;
s->cap = ncap;
}
s->v[s->n++] = n;
}
static void ns_free(nodeset_t *s) {
cbm_free(CBM_MEM_CLASS_OTHER, (void *)s->v);
memset(s, 0, sizeof(*s));
}
/* The qualified name without the project prefix. */
static const char *rel_qn(const pdfr_index_t *x, const char *qn) {
if (qn && x->project_len && strncmp(qn, x->project, x->project_len) == 0 &&
qn[x->project_len] == '.') {
return qn + x->project_len + 1;
}
return qn ? qn : "";
}
static bool code_node(const pdfr_index_t *x, const cbm_gbuf_node_t *n) {
if (!n->label || in_list(n->label, PDFR_EXCLUDED_LABELS) || !n->qualified_name ||
!n->qualified_name[0]) {
return false;
}
if (n->file_path && n->file_path[0] == '<') {
return false; /* builtins */
}
return strncmp(rel_qn(x, n->qualified_name), "__", 2) != 0; /* dependency stand-ins */
}
/* s is a component-aligned suffix of q ('.'-separated) */
static bool comp_suffix(const char *q, const char *s) {
size_t ql = strlen(q);
size_t sl = strlen(s);
if (sl > ql || memcmp(q + ql - sl, s, sl) != 0) {
return false;
}
return sl == ql || q[ql - sl - 1] == '.';
}
/* The owner-qualified name of a node: rel(parent_class) + "." + name, when the
* node has a parent_class (properties JSON). */
static bool owner_name(const pdfr_index_t *x, const cbm_gbuf_node_t *n, char *buf, size_t cap) {
const char *p = n->properties_json ? strstr(n->properties_json, "\"parent_class\":\"") : NULL;
if (!p) {
return false;
}
p += strlen("\"parent_class\":\"");
char pc[PDFR_KEY_CAP];
size_t k = 0;
while (*p && *p != '"' && k + 1 < sizeof(pc)) {
if (*p == '\\' && p[1]) {
p++;
}
pc[k++] = *p++;
}
pc[k] = '\0';
if (*p != '"') {
return false;
}
const char *prel = rel_qn(x, pc);
int w = snprintf(buf, cap, "%s.%s", prel, n->name ? n->name : "");
return w > 0 && (size_t)w < cap;
}
static void qn_candidate(const pdfr_index_t *x, const cbm_gbuf_node_t *n, const char *s,
nodeset_t *out, nodeset_t *exact) {
if (!code_node(x, n)) {
return;
}
const char *rel = rel_qn(x, n->qualified_name);
char alt[PDFR_KEY_CAP];
bool owned = owner_name(x, n, alt, sizeof(alt));
/* A member whose QN leaves its owner out (a Go method's has no receiver)
* is named through the owner: `time.Now` is package time's function,
* never the method Now of a type that package time declares. */
bool ownerless_qn = owned && !comp_suffix(rel, alt);
bool via_rel = !ownerless_qn && comp_suffix(rel, s);
bool via_alt = !via_rel && owned && strcmp(alt, rel) != 0 && comp_suffix(alt, s);
if (via_rel || via_alt) {
ns_add(out, n);
if (via_rel && strcmp(rel, s) == 0) {
ns_add(exact, n);
}
}
}
/* Nodes whose QN or owner-qualified name ends with the mention s (>= 2
* components). A node's name may itself hold dots, so every component suffix
* of s is tried as a name. */
static void qn_suffix_set(const pdfr_index_t *x, const cbm_gbuf_t *graph, const char *s,
nodeset_t *out, nodeset_t *exact) {
const char *starts[PDFR_MAX_COMPS];
int nc = 0;
starts[nc++] = s;
for (const char *q = s; *q && nc < PDFR_MAX_COMPS; q++) {
if (*q == '.') {
starts[nc++] = q + 1;
}
}
if (nc < 2) {
return;
}
for (int j = nc - 1; j >= 0; j--) {
const cbm_gbuf_node_t **nodes = NULL;
int count = 0;
if (cbm_gbuf_find_by_name(graph, starts[j], &nodes, &count) != 0) {
continue;
}
for (int i = 0; i < count; i++) {
qn_candidate(x, nodes[i], s, out, exact);
}
}
for (const pdfr_list_t *l = (const pdfr_list_t *)cbm_ht_get(x->irregular, starts[nc - 1]); l;
l = l->next) {
qn_candidate(x, l->node, s, out, exact);
}
}
static int node_id_cmp(const void *a, const void *b) {
const cbm_gbuf_node_t *x = *(const cbm_gbuf_node_t *const *)a;
const cbm_gbuf_node_t *y = *(const cbm_gbuf_node_t *const *)b;
return x->id < y->id ? -1 : x->id > y->id ? 1 : 0;
}
/* The field test's _collapse: one entity's several nodes become one. */
static const cbm_gbuf_node_t *collapse(const pdfr_index_t *x, nodeset_t *s) {
if (s->n == 1) {
return s->v[0];
}
if (s->n == 0) {
return NULL;
}
qsort((void *)s->v, (size_t)s->n, sizeof(*s->v), node_id_cmp);
const cbm_gbuf_node_t *c = NULL;
int containers = 0;
for (int i = 0; i < s->n; i++) {
if (s->v[i]->label && in_list(s->v[i]->label, PDFR_CONTAINERS)) {
c = s->v[i];
containers++;
}
}
if (containers == 1) {
const char *crel = rel_qn(x, c->qualified_name);
size_t cl = strlen(crel);
bool all = true;
for (int i = 0; i < s->n && all; i++) {
const cbm_gbuf_node_t *n = s->v[i];
if (n == c) {
continue;
}
const char *rel = rel_qn(x, n->qualified_name);
all = n->name && c->name && strcmp(n->name, c->name) == 0 &&
strncmp(rel, crel, cl) == 0 && rel[cl] == '.';
}
if (all) {
return c; /* a class and its constructors */
}
}
const cbm_gbuf_node_t *f = s->v[0];
for (int i = 1; i < s->n; i++) {
const cbm_gbuf_node_t *n = s->v[i];
if (strcmp(n->file_path ? n->file_path : "", f->file_path ? f->file_path : "") != 0 ||
n->start_line != f->start_line ||
strcmp(n->name ? n->name : "", f->name ? f->name : "") != 0) {
return NULL;
}
}
return f; /* duplicate nodes of one definition: the lowest id */
}
/* ── One mention form ────────────────────────────────────────────── */
typedef enum { RES_NONE = 0, RES_AMBIGUOUS, RES_HIT } res_kind_t;
typedef struct {
res_kind_t kind;
bool exact;
const cbm_gbuf_node_t *node;
} res_t;
static res_t outcome(const pdfr_index_t *x, nodeset_t *s, bool exact) {
res_t r = {RES_NONE, false, NULL};
if (s->n == 0) {
return r;
}
const cbm_gbuf_node_t *n = collapse(x, s);
if (n) {
r.kind = RES_HIT;
r.exact = exact;
r.node = n;
} else {
r.kind = RES_AMBIGUOUS;
}
return r;
}
static void list_to_set(const pdfr_list_t *l, nodeset_t *s) {
for (; l; l = l->next) {
ns_add(s, l->node);
}
}
static res_t resolve_form(const pdfr_index_t *x, const cbm_gbuf_t *graph, int syntax,
const char *form) {
res_t none = {RES_NONE, false, NULL};
if (syntax == CBM_DOCLINK_PDF_PATH || syntax == CBM_DOCLINK_PDF_FILE) {
const char *p = form;
while (*p == '/') {
p++;
}
char key[PDFR_KEY_CAP];
size_t n = strlen(p);
while (n > 0 && p[n - 1] == '/') {
n--;
}
if (n == 0 || n >= sizeof(key)) {
return none;
}
memcpy(key, p, n);
key[n] = '\0';
nodeset_t s = {0};
list_to_set((const pdfr_list_t *)cbm_ht_get(x->paths, key), &s);
if (s.n) {
res_t r = outcome(x, &s, true);
ns_free(&s);
return r;
}
const char *slash = strrchr(key, '/');
for (const pdfr_list_t *l =
(const pdfr_list_t *)cbm_ht_get(x->basenames, slash ? slash + 1 : key);
l; l = l->next) {
const char *fp = l->node->file_path;
while (*fp == '/') {
fp++;
}
size_t fl = strlen(fp);
while (fl > 0 && fp[fl - 1] == '/') {
fl--;
}
if (fl >= n && memcmp(fp + fl - n, key, n) == 0 && (fl == n || fp[fl - n - 1] == '/')) {
ns_add(&s, l->node);
}
}
if (s.n) {
res_t r = outcome(x, &s, false);
ns_free(&s);
return r;
}
ns_free(&s);
if (syntax == CBM_DOCLINK_PDF_PATH) {
return none;
}
}
if (syntax == CBM_DOCLINK_PDF_QN || syntax == CBM_DOCLINK_PDF_FILE) {
nodeset_t all = {0};
nodeset_t ex = {0};
qn_suffix_set(x, graph, form, &all, &ex);
res_t r = none;
if (ex.n == 1) {
r.kind = RES_HIT;
r.exact = true;
r.node = ex.v[0];
} else if (all.n) {
r = outcome(x, &all, false);
}
ns_free(&all);
ns_free(&ex);
return r;
}
if (syntax == CBM_DOCLINK_PDF_NAME) {
const cbm_gbuf_node_t **nodes = NULL;
int count = 0;
nodeset_t s = {0};
if (cbm_gbuf_find_by_name(graph, form, &nodes, &count) == 0) {
for (int i = 0; i < count; i++) {
if (code_node(x, nodes[i])) {
ns_add(&s, nodes[i]);
}
}
}
res_t r = outcome(x, &s, false);
ns_free(&s);
return r;
}
return none;
}
/* A join's forms: the first, plus the hyphen-kept alternative. One node hit
* by the forms is the outcome; two different nodes are ambiguous. */
static res_t resolve_join(const pdfr_index_t *x, const cbm_gbuf_t *graph, int syntax,
const char *raw) {
char buf[PDFR_KEY_CAP];
const char *end = strchr(raw, '\x1e');
size_t len = end ? (size_t)(end - raw) : strlen(raw);
if (len >= sizeof(buf)) {
res_t none = {RES_NONE, false, NULL};
return none;
}
memcpy(buf, raw, len);
buf[len] = '\0';
char *forms[PDFR_MAX_FORMS];
int nf = 0;
forms[nf++] = buf;
char *sep = strchr(buf, '\x1f');
if (sep) {
*sep = '\0';
forms[nf++] = sep + 1;
}
res_t first = resolve_form(x, graph, syntax, forms[0]);
if (nf == 1) {
return first;
}
res_t second = resolve_form(x, graph, syntax, forms[1]);
if (first.kind == RES_HIT && second.kind == RES_HIT) {
if (first.node == second.node) {
return first;
}
res_t amb = {RES_AMBIGUOUS, false, NULL};
return amb;
}
if (first.kind == RES_HIT) {
return first;
}
if (second.kind == RES_HIT) {
return second;
}
return first;
}
/* ── Hygiene ─────────────────────────────────────────────────────── */
static bool data_target(const char *fp) {
char low[PDFR_KEY_CAP];
size_t n = strlen(fp);
if (n >= sizeof(low)) {
return false;
}
for (size_t i = 0; i <= n; i++) {
low[i] = (char)(fp[i] >= 'A' && fp[i] <= 'Z' ? fp[i] + ('a' - 'A') : fp[i]);
}
for (int i = 0; PDFR_DATA_EXT[i]; i++) {
if (ends_with(low, PDFR_DATA_EXT[i])) {
return true;
}
}
return false;
}
static bool test_target(const char *fp) {
const char *base = strrchr(fp, '/');
base = base ? base + 1 : fp;
if ((starts_with(base, "test_") && ends_with(base, ".py")) || ends_with(base, "_test.py") ||
ends_with(base, "_test.go") || ends_with(base, "Test.cs") || ends_with(base, "Tests.cs") ||
ends_with(base, "Test.java") || ends_with(base, "Tests.java") ||
strcmp(base, "conftest.py") == 0 || ends_with(base, ".t.sol")) {
return true;
}
static const char *const js[] = {".test.js", ".test.jsx", ".test.ts", ".test.tsx", ".spec.js",
".spec.jsx", ".spec.ts", ".spec.tsx", NULL};
for (int i = 0; js[i]; i++) {
if (ends_with(base, js[i]) && strlen(base) > strlen(js[i])) {
return true;
}
}
const char *s = fp;
for (const char *q = fp; q < base; q++) {
if (*q == '/') {
char comp[PDFR_KEY_CAP];
size_t n = (size_t)(q - s);
if (n < sizeof(comp)) {
memcpy(comp, s, n);
comp[n] = '\0';
if (in_list(comp, PDFR_TEST_DIRS)) {
return true;
}
}
s = q + 1;
}
}
return false;
}
/* ── Per file ────────────────────────────────────────────────────── */
typedef struct {
const CBMDocLink *base;
int n;
uint8_t *suppressed; /* line-local tokens a resolved join replaces */
res_t *join; /* each join token's own outcome */
/* A page set repeats its paths and names (an audit names one file on
* every page): mention text -> 1 + the index of the token whose outcome
* res[] holds, reused for a token of the same family. */
CBMHashTable *memo;
res_t *res;
} pdfr_state_t;
static void pdfr_end(void *state);
static void *pdfr_prepare(const void *index, int run_file, const CBMDocLink *links, int n,
const cbm_gbuf_t *graph) {
(void)run_file;
const pdfr_index_t *x = (const pdfr_index_t *)index;
pdfr_state_t *st = (pdfr_state_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*st));
if (!st) {
return NULL;
}
st->base = links;
st->n = n;
st->suppressed = (uint8_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(n ? n : 1));
st->join = (res_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(n ? n : 1) * sizeof(res_t));
st->res = (res_t *)cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)(n ? n : 1) * sizeof(res_t));
st->memo = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, PDFR_MEMO_INIT);
if (!st->suppressed || !st->join || !st->res || !st->memo) {
pdfr_end(st);
return NULL;
}
for (int i = 0; i < n; i++) {
const CBMDocLink *l = &links[i];
if (!(l->flags & CBM_DOCLINK_FLAG_JOIN) || !l->raw) {
continue;
}
st->join[i] = resolve_join(x, graph, l->syntax, l->raw);
if (st->join[i].kind != RES_HIT) {
continue;
}
const char *f = strchr(l->raw, '\x1e');
while (f && *f) {
f++;
char *endp = NULL;
long k = strtol(f, &endp, 10);
if (endp == f) {
break;
}
if (k >= 0 && k < n) {
st->suppressed[k] = 1;
}
f = (*endp == ',') ? endp : NULL;
}
}
return st;
}
static void pdfr_end(void *state) {
pdfr_state_t *st = (pdfr_state_t *)state;
if (!st) {
return;
}
cbm_free(CBM_MEM_CLASS_OTHER, st->suppressed);
cbm_free(CBM_MEM_CLASS_OTHER, st->join);
cbm_free(CBM_MEM_CLASS_OTHER, st->res);
cbm_ht_free(st->memo);
cbm_free(CBM_MEM_CLASS_OTHER, st);
}
/* The mention's first component names something of this repository. */
static bool first_known(const pdfr_index_t *x, const cbm_gbuf_t *graph, int syntax,
const char *raw) {
char first[PDFR_KEY_CAP];
size_t n = 0;
const char *p = raw;
while (*p == '/') {
p++;
}
char sep = syntax == CBM_DOCLINK_PDF_PATH ? '/' : '.';
while (p[n] && p[n] != sep && n + 1 < sizeof(first)) {
first[n] = p[n];
n++;
}
first[n] = '\0';
if (!n) {
return false;
}
if (cbm_ht_get(x->dirs, first) || cbm_ht_get(x->basenames, first)) {
return true;
}
const cbm_gbuf_node_t **nodes = NULL;
int count = 0;
return cbm_gbuf_find_by_name(graph, first, &nodes, &count) == 0 && count > 0;
}
static void finish_hit(cbm_doclink_outcome_t *out, res_t r) {
const char *fp = r.node->file_path ? r.node->file_path : "";
if (data_target(fp)) {
out->kind = CBM_DOCLINK_LOCAL; /* R2: a data file, not code */
return;
}
if (test_target(fp)) {
out->kind = CBM_DOCLINK_UNRESOLVED; /* R4 */
out->reason = CBM_DOCLINK_REASON_TEST_ONLY;
return;
}
out->kind = CBM_DOCLINK_EDGE;
out->target = r.node;
out->exact = r.exact;
}
static void pdfr_resolve(const void *index, void *state, int run_file, const CBMDocLink *link,
const cbm_gbuf_t *graph, cbm_doclink_outcome_t *out) {
(void)run_file;
const pdfr_index_t *x = (const pdfr_index_t *)index;
pdfr_state_t *st = (pdfr_state_t *)state;
out->kind = CBM_DOCLINK_LOCAL;
if (!x || !link->raw) {
return;
}
int i = st ? (int)(link - st->base) : -1;
bool in_file = st && i >= 0 && i < st->n;
if (link->flags & CBM_DOCLINK_FLAG_JOIN) {
/* a join stands only when it resolved; a bare-name join never links */
if (in_file && st->join[i].kind == RES_HIT && link->syntax != CBM_DOCLINK_PDF_NAME) {
finish_hit(out, st->join[i]);
}
return;
}
if (in_file && st->suppressed[i]) {
return; /* a resolved join replaced this fragment */
}
if (link->syntax == CBM_DOCLINK_PDF_NAME) {
return;
}
res_t r;
uintptr_t seen = in_file ? (uintptr_t)cbm_ht_get(st->memo, link->raw) : 0;
if (seen && st->base[seen - 1].syntax == link->syntax) {
r = st->res[seen - 1];
} else {
r = resolve_form(x, graph, link->syntax, link->raw);
if (in_file && !seen) {
st->res[i] = r;
cbm_ht_set(st->memo, link->raw, (void *)(uintptr_t)(i + 1));
}
}
if (r.kind == RES_HIT) {
finish_hit(out, r);
} else if (r.kind == RES_AMBIGUOUS) {
out->kind = CBM_DOCLINK_UNRESOLVED;
out->reason = CBM_DOCLINK_REASON_AMBIGUOUS;
} else if (first_known(x, graph, link->syntax, link->raw)) {
out->kind = CBM_DOCLINK_UNRESOLVED;
out->reason = CBM_DOCLINK_REASON_MISSING;
}
}
const cbm_doclink_resolver_t cbm_doclink_pdf_resolver = {
.langs = {CBM_LANG_PDF},
.lang_count = 1,
.scope_tag = NULL,
.build = pdfr_build,
.destroy = pdfr_destroy,
.file_prepare = pdfr_prepare,
.file_end = pdfr_end,
.resolve = pdfr_resolve,
.via = "pdf",
};
src/pipeline/doc_links_rst.c
+1792
View File
File diff suppressed because it is too large Load Diff
src/pipeline/pass_definitions.c
+4
View File
@@ -323,6 +323,10 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
append_json_string(buf, bufsize, &pos, "route_method", def->route_method);
append_json_string(buf, bufsize, &pos, "http_client", def->http_client);
append_json_string(buf, bufsize, &pos, "http_base_url", def->http_base_url);
/* A document definition's facts (an ADR's id, status, date, ...): key, value pairs. */
for (const char **kv = def->extra_props; kv && kv[0] && kv[SKIP_ONE]; kv += PAIR_LEN) {
append_json_string(buf, bufsize, &pos, kv[0], kv[SKIP_ONE]);
}
/* MinHash fingerprint — append if present and buffer has room. */
if (def->fingerprint && def->fingerprint_k > 0 &&
src/pipeline/pass_parallel.c
+13 -2
View File
@@ -551,6 +551,10 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def)
append_json_string(buf, bufsize, &pos, "route_method", def->route_method);
append_json_string(buf, bufsize, &pos, "http_client", def->http_client);
append_json_string(buf, bufsize, &pos, "http_base_url", def->http_base_url);
/* A document definition's facts (an ADR's id, status, date, ...): key, value pairs. */
for (const char **kv = def->extra_props; kv && kv[0] && kv[SKIP_ONE]; kv += PAIR_LEN) {
append_json_string(buf, bufsize, &pos, kv[0], kv[SKIP_ONE]);
}
/* MinHash fingerprint — append if present and buffer has room.
* Hex-encoded K=64 uint32 = 512 chars + key/quotes ≈ 520 chars. */
@@ -3551,7 +3555,13 @@ static void resolve_file_usages(resolve_ctx_t *rc, resolve_worker_state_t *ws,
imp_keys, imp_count)) {
continue;
}
tgt = cbm_gbuf_find_by_qn(rc->main_gbuf, res.qualified_name);
/* a plain value reference names the field twin of a same-named
* method; a callable-reference candidate keeps the method */
bool value_ref = usage->kind == CBM_USAGE_VALUE && !usage->may_be_call_reference;
tgt = cbm_gbuf_find_by_qn(
rc->main_gbuf, value_ref
? cbm_registry_value_target(rc->registry, res.qualified_name)
: res.qualified_name);
/* #1928: the registry fallback is a bare-name guess — never let it
* bind a reference across a language boundary. Mirrors the
* sequential twin (pass_usages.c). */
@@ -3672,7 +3682,8 @@ static void resolve_file_rw(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFi
if (!res.qualified_name || res.qualified_name[0] == '\0') {
continue;
}
const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_qn(rc->main_gbuf, res.qualified_name);
const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_qn(
rc->main_gbuf, cbm_registry_value_target(rc->registry, res.qualified_name));
if (!tgt || src->id == tgt->id) {
continue;
}
src/pipeline/pass_usages.c
+8 -2
View File
@@ -255,7 +255,12 @@ static int resolve_usage_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *res
imp_keys, imp_count)) {
continue;
}
tgt = cbm_gbuf_find_by_qn(ctx->gbuf, res.qualified_name);
/* a plain value reference names the field twin of a same-named
* method; a callable-reference candidate keeps the method */
bool value_ref = usage->kind == CBM_USAGE_VALUE && !usage->may_be_call_reference;
tgt = cbm_gbuf_find_by_qn(
ctx->gbuf, value_ref ? cbm_registry_value_target(ctx->registry, res.qualified_name)
: res.qualified_name);
/* #1928: the registry fallback is a bare-name guess — never let it
* bind a reference across a language boundary (the LSP-backed
* semantic branch above is not affected). */
@@ -381,7 +386,8 @@ static int resolve_rw_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *result
continue;
}
const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_qn(ctx->gbuf, res.qualified_name);
const cbm_gbuf_node_t *tgt = cbm_gbuf_find_by_qn(
ctx->gbuf, cbm_registry_value_target(ctx->registry, res.qualified_name));
if (!tgt || src->id == tgt->id) {
continue;
}
src/pipeline/pipeline.c
+1 -1
View File
@@ -1896,7 +1896,7 @@ static void log_result_census(const char *tag, CBMFileResult **cache, int file_c
str_def_fp += def->fingerprint ? (size_t)def->fingerprint_k * sizeof(uint32_t) : 0;
str_def_misc += census_len(def->route_path) + census_len(def->route_method) +
census_len(def->impl_trait) + census_len(def->http_base_url) +
census_len(def->decl_namespace);
census_list(def->extra_props) + census_len(def->decl_namespace);
}
for (int c = 0; c < r->calls.count; c++) {
const CBMCall *call = &r->calls.items[c];
src/pipeline/pipeline.h
+6
View File
@@ -440,6 +440,12 @@ const char *cbm_registry_unique_field_qn(const cbm_registry_t *r, const char *me
/* Get the label of a qualified name, or NULL if not found. */
const char *cbm_registry_label_of(const cbm_registry_t *r, const char *qn);
/* A read, a write or a plain value reference never denotes a call: when `qn`
* is a Method whose class body also declares a same-named field (the extractor
* fences that field as `<qn>#field`), the field is the target. Returns the
* field's QN, else `qn`. Pure; unit-tested in test_registry.c. */
const char *cbm_registry_value_target(const cbm_registry_t *r, const char *qn);
/* Find all QNs with a given simple name. Sets *out and *count.
* Caller does NOT free the array (owned by registry). */
int cbm_registry_find_by_name(const cbm_registry_t *r, const char *name, const char ***out,
src/pipeline/pipeline_delta.c
+69 -4
View File
@@ -37,6 +37,7 @@
#include <string.h>
#include "foundation/compat_fs.h"
#include "foundation/mem_core.h"
#include "foundation/log.h"
#include "sqlite3.h"
#include "store/store.h"
@@ -254,6 +255,50 @@ int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const *
* guessing. The proxy load stays edge-free and property-free, which is
* where the old full load actually spent its time. */
/* `qn` and `parent` without the project prefix. */
static const char *delta_rel(const char *project, const char *qn) {
size_t pl = project ? strlen(project) : 0;
if (qn && pl && strncmp(qn, project, pl) == 0 && qn[pl] == '.') {
return qn + pl + 1;
}
return qn ? qn : "";
}
/* A proxy's properties carrying `parent` when owner + "." + name is not the
* node's own (relative) QN, merged with the http client keys if any; NULL
* when the owner adds nothing (CBM_MEM_CLASS_OTHER; the caller frees). */
static char *delta_owner_props(const char *project, const char *name, const char *qn,
const char *parent, const char *client_props) {
const char *rel = delta_rel(project, qn);
const char *prel = delta_rel(project, parent);
size_t pl = strlen(prel);
size_t nl = name ? strlen(name) : 0;
if (strlen(rel) == pl + 1 + nl && strncmp(rel, prel, pl) == 0 && rel[pl] == '.' &&
(nl == 0 || memcmp(rel + pl + 1, name, nl) == 0)) {
return NULL;
}
size_t cap = 2 * strlen(parent) + (client_props ? strlen(client_props) : 0) + 32;
char *out = (char *)cbm_alloc(CBM_MEM_CLASS_OTHER, cap);
if (!out) {
return NULL;
}
size_t k = 0;
k += (size_t)snprintf(out + k, cap - k, "{\"parent_class\":\"");
for (const char *p = parent; *p && k + 3 < cap; p++) {
if (*p == '"' || *p == '\\') {
out[k++] = '\\';
}
out[k++] = *p;
}
out[k++] = '"';
if (client_props && client_props[0] == '{' && client_props[1] && client_props[1] != '}') {
snprintf(out + k, cap - k, ",%s", client_props + 1);
} else {
snprintf(out + k, cap - k, "}");
}
return out;
}
int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf) {
sqlite3 *db = cbm_store_get_db(store);
if (!db) {
@@ -292,7 +337,13 @@ int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *g
* compose `api.get('/p')` against its unchanged wrapper exactly as a full
* build does. Only the two keys, only on the rare Module/Variable rows
* that carry them; every other proxy stays "{}". Proxies are never
* written back (cbm_delta_patch skips id <= max_db_id). */
* written back (cbm_delta_patch skips id <= max_db_id).
*
* The other: a node's parent_class where it changes the name the node
* goes by -- a Go method's QN has no receiver, a class-body variable's QN
* no class. A document naming `Order.Reset` binds such a node through
* its owner (doc_links_pdf.c), so a re-resolved document needs it too.
* Only where owner + name differs from the QN (delta_owner_props). */
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(
db,
@@ -300,7 +351,11 @@ int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *g
" CASE WHEN label IN ('Module','Variable')"
" AND instr(properties, '\"http_client\"') > 0"
" THEN json_object('http_client', json_extract(properties, '$.http_client'),"
" 'http_base_url', json_extract(properties, '$.http_base_url')) END"
" 'http_base_url', json_extract(properties, '$.http_base_url')) END,"
" start_line, end_line,"
" CASE WHEN label IN ('Method','Variable')"
" AND instr(properties, '\"parent_class\"') > 0"
" THEN json_extract(properties, '$.parent_class') END"
" FROM nodes"
" WHERE project = ?1 AND label NOT IN"
" ('Macro','Comment','Section','Branch','Commit','Tag')"
@@ -318,10 +373,20 @@ int64_t cbm_delta_preseed(cbm_store_t *store, const char *project, cbm_gbuf_t *g
const char *qn = (const char *)sqlite3_column_text(stmt, 3);
const char *fp = (const char *)sqlite3_column_text(stmt, 4);
const char *client_props = (const char *)sqlite3_column_text(stmt, 5);
/* Lines too: a document reference that names a line range binds the
* definition holding it (doc_links_md.c), in a re-resolved document as
* in a full build. */
int start_line = sqlite3_column_int(stmt, 6);
int end_line = sqlite3_column_int(stmt, 7);
const char *parent = (const char *)sqlite3_column_text(stmt, 8);
char *owner_props =
parent ? delta_owner_props(project, name, qn, parent, client_props) : NULL;
/* Pin the gbuf id to the database id: proxies ARE their rows. */
cbm_gbuf_set_next_id(gbuf, id);
int64_t got = cbm_gbuf_upsert_node(gbuf, label, name, qn, fp ? fp : "", 0, 0,
client_props ? client_props : "{}");
int64_t got =
cbm_gbuf_upsert_node(gbuf, label, name, qn, fp ? fp : "", start_line, end_line,
owner_props ? owner_props : (client_props ? client_props : "{}"));
cbm_free(CBM_MEM_CLASS_OTHER, owner_props);
if (got != id) {
/* A QN collision inside the preseed set would silently split
* identity between RAM and disk; the run cannot be trusted. */
src/pipeline/pipeline_incremental.c
+38 -7
View File
@@ -2363,6 +2363,12 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p
char **changed_paths = NULL;
char **dependents = NULL;
int dependent_count = 0;
/* body-edited files (surface unchanged) and the documents bound to their
* lines, which re-resolve with them */
const char **body_targets = NULL;
int body_target_count = 0;
char **line_docs = NULL;
int line_doc_count = 0;
cbm_lsp_surface_row_t *probe_rows = NULL;
int probe_count = 0;
cbm_file_info_t *probe_files = NULL;
@@ -2529,13 +2535,15 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p
/* Early cutoff per changed file; surface-changed files seed the
* dependent query. Deleted files always seed it. */
int n_surface_changed = 0;
/* dep_targets, then the body-edited files (body_targets) in the same block */
const char **dep_targets =
(const char **)calloc((size_t)(n_changed + n_deleted), sizeof(char *));
(const char **)calloc((size_t)(n_changed + n_deleted + n_changed), sizeof(char *));
if (!dep_targets) {
cbm_ht_free(rows_by_path);
decline = "alloc";
goto done;
}
body_targets = dep_targets + n_changed + n_deleted;
int dep_target_count = 0;
for (int i = 0; i < n_changed; i++) {
const cbm_lsp_surface_row_t *stored_row = cbm_ht_get(rows_by_path, changed_paths[i]);
@@ -2553,7 +2561,10 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p
goto done;
}
if (strcmp(stored_row->surface_sha, fresh_row->surface_sha) == 0) {
continue; /* body edit: the file re-resolves, nobody else does */
/* body edit: the file re-resolves, and of the others only the
* documents bound to its lines (below) */
body_targets[body_target_count++] = changed_paths[i];
continue;
}
bool added = false;
const char *why = NULL;
@@ -2583,14 +2594,23 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p
}
cbm_ht_free(rows_by_path);
if (dep_target_count > 0 &&
cbm_store_get_dependent_files(store, project, dep_targets, dep_target_count, &dependents,
&dependent_count) != CBM_STORE_OK) {
free(dep_targets);
/* A body edit keeps a file's names but moves its lines: a document that
* bound a reference by lines (`file#L3-L9`, :lines:) would keep a stale
* target, so the documents with edges into a body-edited file re-resolve
* too (added to the closure below). */
bool dep_failed =
(dep_target_count > 0 &&
cbm_store_get_dependent_files(store, project, dep_targets, dep_target_count, &dependents,
&dependent_count) != CBM_STORE_OK) ||
(body_target_count > 0 &&
cbm_store_get_dependent_files(store, project, body_targets, body_target_count, &line_docs,
&line_doc_count) != CBM_STORE_OK);
free(dep_targets); /* body_targets with it */
body_targets = NULL;
if (dep_failed) {
decline = "dependent_query_failed";
goto done;
}
free(dep_targets);
if (gone_scope_changed) {
decline = "doc_scope_changed";
goto done;
@@ -2618,6 +2638,16 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p
if (decline) {
goto done;
}
/* the documents bound to a body-edited file's lines; code files stay out:
* their references bind names, which a body edit keeps */
for (int i = 0; i < line_doc_count; i++) {
const cbm_file_info_t *info = cbm_ht_get(files_by_path, line_docs[i]);
if (info && cbm_doclinks_binds_lines(info->language) &&
!cbm_ht_get(closure_set, line_docs[i])) {
cbm_ht_set(closure_set, line_docs[i], line_docs[i]);
n_row_dependents++;
}
}
/* Files whose unresolved doc-link references name something a changed or
* deleted file no longer declares re-resolve too. */
doc_row_name_dependents(plan->doc_rows, plan->doc_row_count, removed_names, row_deps);
@@ -2665,6 +2695,7 @@ done:
free(probe_files);
cbm_store_free_lsp_surfaces(probe_rows, probe_count);
cbm_store_free_dependent_files(dependents, dependent_count);
cbm_store_free_dependent_files(line_docs, line_doc_count);
free(changed_paths);
cbm_ht_free(fresh_by_path);
cbm_ht_free(files_by_path);
src/pipeline/pipeline_internal.h
+7 -2
View File
@@ -1269,8 +1269,13 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr
* QNs for every unchanged file, so it is rebuilt in full once.
* 5: doc-comment references became MENTIONS edges and doc_link_unresolved
* rows, and C# LSP surfaces carry the doc-link scope ("dl"); an index
* built before has neither, so it rebuilds once on upgrade. */
enum { CBM_SEMANTIC_INDEX_VERSION = 5 };
* built before has neither, so it rebuilds once on upgrade.
* 6: documents became link sources: Markdown, reStructuredText and AsciiDoc
* sections and PDF pages hold MENTIONS edges into the code, ADR records
* are nodes, and AsciiDoc files, PDFs and a Sphinx project's extra
* source suffixes are indexed. An index built before has none of this
* for its unchanged documents, so it rebuilds once on upgrade. */
enum { CBM_SEMANTIC_INDEX_VERSION = 6 };
typedef struct {
cbm_gbuf_t *gbuf;
src/pipeline/registry.c
+14
View File
@@ -1237,6 +1237,20 @@ const char *cbm_registry_label_of(const cbm_registry_t *r, const char *qn) {
return cbm_ht_get(r->exact, qn);
}
const char *cbm_registry_value_target(const cbm_registry_t *r, const char *qn) {
const char *label = cbm_registry_label_of(r, qn);
if (!label || strcmp(label, "Method") != 0) {
return qn;
}
char twin[CBM_SZ_1K];
int n = snprintf(twin, sizeof(twin), "%s#field", qn);
if (n < 0 || (size_t)n >= sizeof(twin)) {
return qn;
}
const char *owned = cbm_ht_get_key(r->exact, twin);
return owned ? owned : qn;
}
int cbm_registry_find_by_name(const cbm_registry_t *r, const char *name, const char ***out,
int *count) {
if (!r || !out || !count) {
tests/repro/repro_call_node_manifest.c
+6 -3
View File
@@ -547,9 +547,12 @@ TEST(repro_call_node_manifest_live_specs_contain_only_expected_primary_entries)
CBMLanguage language = (CBMLanguage)value;
const CBMLangSpec *spec = cbm_lang_spec(language);
/* The registry reproduction owns Nim's stale enum and Studio Export's
* transform-only, intentionally grammar-free classification. */
if (language == CBM_LANG_NIM || language == CBM_LANG_OBJECTSCRIPT_EXPORT) {
/* The registry reproduction owns Nim's stale enum and the
* intentionally grammar-free classifications: Studio Export's
* transform-only one and the DOCUMENT ones of PDF and AsciiDoc
* (doc_pdf.c, doc_adoc.c). */
if (language == CBM_LANG_NIM || language == CBM_LANG_OBJECTSCRIPT_EXPORT ||
language == CBM_LANG_PDF || language == CBM_LANG_ASCIIDOC) {
continue;
}
if (!spec || spec->language != language) {
tests/repro/repro_language_registry.c
+18 -7
View File
@@ -18,6 +18,7 @@ typedef enum {
CAP_NO_CALL,
CAP_TRANSFORM_ONLY,
CAP_UNSUPPORTED,
CAP_DOCUMENT,
CAP_COUNT
} LanguageCapability;
@@ -43,6 +44,9 @@ size_t repro_call_argument_matrix_b_copy_language_ids(CBMLanguage *language_ids,
"source is transformed into another registered language first"}
#define UNSUPPORTED(lang) \
[CBM_LANG_##lang] = {CAP_UNSUPPORTED, #lang, "enum retained without a registered grammar"}
#define DOCUMENT(lang) \
[CBM_LANG_##lang] = {CAP_DOCUMENT, #lang, \
"a document format read without a grammar; no code, no calls"}
/* Explicit ledger: no default initializer is intentional. A new enum remains
* CAP_UNSET until its call-metadata/reference-vocabulary behavior is audited. */
@@ -213,6 +217,8 @@ static const LanguageCapabilityEntry LANGUAGE_CAPABILITIES[CBM_LANG_COUNT] = {
CALL_WITH_REFERENCE_VOCAB(ARKTS),
CALL_WITHOUT_REFERENCE_VOCAB(PLSQL),
CALL_WITHOUT_REFERENCE_VOCAB(CHIALISP),
DOCUMENT(PDF),
DOCUMENT(ASCIIDOC),
};
#undef CALL_WITH_REFERENCE_VOCAB
@@ -220,6 +226,7 @@ static const LanguageCapabilityEntry LANGUAGE_CAPABILITIES[CBM_LANG_COUNT] = {
#undef NO_CALL
#undef TRANSFORM_ONLY
#undef UNSUPPORTED
#undef DOCUMENT
_Static_assert(sizeof(LANGUAGE_CAPABILITIES) / sizeof(LANGUAGE_CAPABILITIES[0]) == CBM_LANG_COUNT,
"language capability ledger must track CBM_LANG_COUNT");
@@ -271,7 +278,8 @@ TEST(repro_language_capability_ledger_covers_every_enum) {
}
counts[entry->capability]++;
if (entry->capability == CAP_UNSUPPORTED || entry->capability == CAP_TRANSFORM_ONLY) {
if (entry->capability == CAP_UNSUPPORTED || entry->capability == CAP_TRANSFORM_ONLY ||
entry->capability == CAP_DOCUMENT) {
if (spec) {
fprintf(stderr,
" [language-registry] lang=%s "
@@ -303,12 +311,14 @@ TEST(repro_language_capability_ledger_covers_every_enum) {
if (counts[CAP_CALL_WITH_REFERENCE_VOCAB] != 88 ||
counts[CAP_CALL_WITHOUT_REFERENCE_VOCAB] != 27 || counts[CAP_NO_CALL] != 49 ||
counts[CAP_TRANSFORM_ONLY] != 1 || counts[CAP_UNSUPPORTED] != 1) {
counts[CAP_TRANSFORM_ONLY] != 1 || counts[CAP_UNSUPPORTED] != 1 ||
counts[CAP_DOCUMENT] != 2) {
fprintf(stderr,
" [language-registry] invariant=capability_partition call_ref_vocab=%d ref_gap=%d "
"no_call=%d transform_only=%d unsupported=%d\n",
"no_call=%d transform_only=%d unsupported=%d document=%d\n",
counts[CAP_CALL_WITH_REFERENCE_VOCAB], counts[CAP_CALL_WITHOUT_REFERENCE_VOCAB],
counts[CAP_NO_CALL], counts[CAP_TRANSFORM_ONLY], counts[CAP_UNSUPPORTED]);
counts[CAP_NO_CALL], counts[CAP_TRANSFORM_ONLY], counts[CAP_UNSUPPORTED],
counts[CAP_DOCUMENT]);
failures++;
}
@@ -321,7 +331,8 @@ TEST(repro_call_argument_matrices_equal_call_capability_ledger) {
EXPECTED_MATRIX_A_ROWS = 68,
EXPECTED_MATRIX_B_ROWS = 49,
EXPECTED_CALL_CAPABLE_LANGUAGES = 115,
EXPECTED_NON_CALL_LANGUAGES = 51,
EXPECTED_NON_CALL_LANGUAGES =
53, /* 49 no-call + transform-only + unsupported + PDF, AsciiDoc */
EXPECTED_NON_CALL_DOMAIN_CONTROLS = 2,
};
CBMLanguage matrix_a_ids[CBM_LANG_COUNT];
@@ -432,11 +443,11 @@ TEST(repro_call_argument_matrices_equal_call_capability_ledger) {
}
if (entry->capability != CAP_NO_CALL && entry->capability != CAP_TRANSFORM_ONLY &&
entry->capability != CAP_UNSUPPORTED) {
entry->capability != CAP_UNSUPPORTED && entry->capability != CAP_DOCUMENT) {
fprintf(stderr,
" [language-registry] lang=%s id=%d "
"invariant=remaining_capability_explicit "
"expected=NO_CALL_or_TRANSFORM_ONLY_or_UNSUPPORTED "
"expected=NO_CALL_or_TRANSFORM_ONLY_or_UNSUPPORTED_or_DOCUMENT "
"actual=%d\n",
language_name, language_id, (int)entry->capability);
failures++;
tests/test_doc_links_adoc.c
+289
View File
@@ -0,0 +1,289 @@
/*
* test_doc_links_adoc.c — AsciiDoc documents -> Sections and MENTIONS edges
* (doc_adoc.c, doc_links_adoc.c).
*
* Headings and delimited blocks, the tokens (include with the page's
* attributes, attribute references, monospace spans), the Antora scope blob,
* and the resolver: resource IDs and collector scans, tag regions read from
* the included file, lines, javadoc attributes to types and members, the
* monospace spans through the Markdown rules, incremental == full.
*/
#include "../src/foundation/compat.h"
#include "test_framework.h"
#include "test_helpers.h"
#include "test_doc_mentions_helpers.h"
#include "cbm.h"
#include "doclink.h"
#include "pipeline/pipeline.h"
#include "pipeline/pipeline_internal.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
static const char *const ADOC_PAGE =
"= Writing Tests\n" /* 1 */
":testDir: example$java\n" /* 2 */
"\n" /* 3 */
"Intro with {Assertions} and {nbsp}.\n" /* 4 */
"\n" /* 5 */
"== First Test\n" /* 6 */
"\n" /* 7 */
"[source,java]\n" /* 8 */
"----\n" /* 9 */
"include::{testDir}/example/FirstTests.java[tags=user_guide]\n" /* 10 */
"== Not A Heading\n" /* 11 */
"----\n" /* 12 */
"\n" /* 13 */
"////\n" /* 14 */
"include::example$java/Hidden.java[]\n" /* 15 */
"////\n" /* 16 */
"// include::example$java/Commented.java[]\n" /* 17 */
"\n" /* 18 */
"=== Details\n" /* 19 */
"\n" /* 20 */
"See `src/main/java/org/x/Assertions.java` and `org.x.Assertions`.\n" /* 21 */
"include::../../../../src/main/java/org/x/Calc.java[lines=4..6]\n" /* 22 */
"include::{testDir}/example/WithImports.java[tag=all]\n"; /* 23 */
static const CBMDefinition *adoc_def(const CBMFileResult *r, const char *name) {
for (int i = 0; i < r->defs.count; i++) {
const CBMDefinition *d = &r->defs.items[i];
if (d->label && strcmp(d->label, "Section") == 0 && d->name && strcmp(d->name, name) == 0) {
return d;
}
}
return NULL;
}
static bool adoc_raw_has(const CBMFileResult *r, const char *needle) {
for (int i = 0; i < r->doc_links.count; i++) {
if (strstr(r->doc_links.items[i].raw, needle)) {
return true;
}
}
return false;
}
TEST(adoc_scan_structure) {
CBMFileResult *r =
dm_extract(ADOC_PAGE, CBM_LANG_ASCIIDOC, "docs/modules/ROOT/pages/writing.adoc");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->doc_links.failed);
const CBMDefinition *top = adoc_def(r, "Writing Tests");
const CBMDefinition *first = adoc_def(r, "First Test");
const CBMDefinition *details = adoc_def(r, "Details");
ASSERT_NOT_NULL(top);
ASSERT_NOT_NULL(first);
ASSERT_NOT_NULL(details);
ASSERT_NULL(adoc_def(r, "Not A Heading")); /* inside a listing block */
ASSERT_EQ(first->start_line, 6);
ASSERT_EQ(first->end_line, 17);
ASSERT_STR_EQ(details->qualified_name, "p.docs.modules.ROOT.pages.writing.Details");
/* an include with the page's attribute substituted, its tags */
const CBMDocLink *inc =
dm_find_token(r, "include::{testDir}/example/FirstTests.java[tags=user_guide]\t"
"example$java/example/FirstTests.java\tuser_guide\t");
ASSERT_NOT_NULL(inc);
ASSERT_EQ(inc->syntax, CBM_DOCLINK_ADOC_INCLUDE);
ASSERT_EQ(inc->line, 10);
ASSERT_STR_EQ(inc->source_qn, first->qualified_name);
ASSERT_NOT_NULL(dm_find_token(r,
"include::../../../../src/main/java/org/x/Calc.java[lines=4..6]"
"\t../../../../src/main/java/org/x/Calc.java\t\t4..6"));
/* nothing from a comment block or a line comment */
ASSERT_FALSE(adoc_raw_has(r, "Hidden.java"));
ASSERT_FALSE(adoc_raw_has(r, "Commented.java"));
/* attribute references: built-ins are none; the page's own carry its value */
ASSERT_NOT_NULL(dm_find_token(r, "{Assertions}\tAssertions\t\t"));
ASSERT_FALSE(adoc_raw_has(r, "{nbsp}"));
/* monospace spans */
ASSERT_EQ(dm_find_token(r, "src/main/java/org/x/Assertions.java")->syntax,
CBM_DOCLINK_ADOC_CODE_PATH);
ASSERT_EQ(dm_find_token(r, "org.x.Assertions")->syntax, CBM_DOCLINK_ADOC_CODE_NAME);
cbm_free_result(r);
PASS();
}
static const char *const ANTORA_YML =
"name: guide\n"
"version: true\n"
"ext:\n"
" collector:\n"
" scan:\n"
" - dir: ./build/generated\n"
" clean: true\n"
" - dir: ./src/test\n"
" into: modules/ROOT/examples\n"
"asciidoc:\n"
" attributes:\n"
" javadoc-root: \"xref:attachment$api\"\n"
" # API\n"
" Assertions: '{javadoc-root}/org.x/org/x/Assertions.html[Assertions]'\n"
" assertEquals: '{javadoc-root}/org.x/org/x/Assertions.html#assertEquals(int,int)[x]'\n"
" Missing: '{javadoc-root}/org.x/org/x/Gone.html[Gone]'\n";
TEST(adoc_antora_scope_blob) {
CBMFileResult *r = dm_extract(ANTORA_YML, CBM_LANG_YAML, "docs/antora.yml");
ASSERT_NOT_NULL(r);
ASSERT_NOT_NULL(r->doc_scope);
ASSERT_STR_EQ(r->doc_scope,
"ad1\n"
"C\n"
"N\tguide\n"
"D\t./build/generated\n"
"D\t./src/test\n"
"I\tmodules/ROOT/examples\n"
"A\tjavadoc-root\txref:attachment$api\n"
"A\tAssertions\t{javadoc-root}/org.x/org/x/Assertions.html[Assertions]\n"
"A\tassertEquals\t{javadoc-root}/org.x/org/x/Assertions.html#assertEquals(int,"
"int)[x]\n"
"A\tMissing\t{javadoc-root}/org.x/org/x/Gone.html[Gone]\n");
cbm_free_result(r);
r = dm_extract("site:\n title: x\n", CBM_LANG_YAML, "config/other.yml");
ASSERT_NOT_NULL(r);
ASSERT_NULL(r->doc_scope);
cbm_free_result(r);
PASS();
}
static void adoc_write_repo(const char *repo, const char *first_tests) {
th_write_file(TH_PATH(repo, "docs/antora.yml"), ANTORA_YML);
th_write_file(TH_PATH(repo, "docs/modules/ROOT/pages/writing.adoc"), ADOC_PAGE);
th_write_file(TH_PATH(repo, "docs/src/test/java/example/FirstTests.java"), first_tests);
th_write_file(TH_PATH(repo, "src/main/java/org/x/Assertions.java"),
"package org.x;\n"
"\n"
"public class Assertions {\n"
" public static void assertEquals(int a, int b) {}\n"
" public static void fail() {}\n"
"}\n");
/* a tag region that takes the imports with the class: it binds the class */
th_write_file(TH_PATH(repo, "docs/src/test/java/example/WithImports.java"),
"package example;\n"
"\n"
"// tag::all[]\n"
"import java.util.List;\n"
"\n"
"class WithImports {\n"
" List<String> names() { return null; }\n"
"}\n"
"// end::all[]\n");
th_write_file(TH_PATH(repo, "src/main/java/org/x/Calc.java"),
"package org.x;\n"
"\n"
"public class Calc {\n"
" public int add(int a, int b) {\n"
" return a + b;\n"
" }\n"
"}\n");
}
static const char *const FIRST_TESTS = "package example;\n" /* 1 */
"\n" /* 2 */
"class FirstTests {\n" /* 3 */
" void helper() {}\n" /* 4 */
"\n" /* 5 */
" // tag::user_guide[]\n" /* 6 */
" void addition() {\n" /* 7 */
" helper();\n" /* 8 */
" }\n" /* 9 */
" // end::user_guide[]\n" /* 10 */
"}\n"; /* 11 */
static bool adoc_edge(const char *db, const char *src, const char *tgt, const char *needle) {
char props[512];
int n = 0;
dm_edge(db, src, tgt, props, sizeof(props), &n);
return n == 1 && (!needle || strstr(props, needle));
}
TEST(adoc_links_pipeline) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dladoc_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char repo[512];
char db[512];
snprintf(repo, sizeof(repo), "%s/repo", tmp);
snprintf(db, sizeof(db), "%s/g.db", tmp);
adoc_write_repo(repo, FIRST_TESTS);
ASSERT_EQ(dm_index(repo, db, NULL), 0);
const char *page = "docs.modules.ROOT.pages.writing";
char src[256];
/* a resource ID through a collector scan; the tag region's innermost method */
snprintf(src, sizeof(src), "%s.First-Test", page);
ASSERT_TRUE(adoc_edge(db, src, "example.FirstTests.addition",
"{\"via\":\"asciidoc\",\"syntax\":\"include\",\"tier\":\"exact\""));
ASSERT_TRUE(adoc_edge(db, src, "example.FirstTests.addition", "\"target_lines\":[7,9]"));
/* a javadoc attribute (under the document title): the type it documents */
snprintf(src, sizeof(src), "%s.Writing-Tests", page);
ASSERT_TRUE(adoc_edge(db, src, "org.x.Assertions", "\"syntax\":\"attribute\""));
/* lines= and a relative include */
snprintf(src, sizeof(src), "%s.Details", page);
ASSERT_TRUE(adoc_edge(db, src, "org.x.Calc.add", "\"target_lines\":[4,6]"));
/* the region's imports are no definition: the one it holds whole binds */
ASSERT_TRUE(adoc_edge(db, src, "example.WithImports", "\"target_lines\":[4,8]"));
/* monospace spans through the Markdown rules */
ASSERT_TRUE(adoc_edge(db, src, "src.main.java.org.x.Assertions.java.__file__",
"\"syntax\":\"code_path\""));
/* back navigation: the method knows the section that includes it */
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN "
"nodes t ON t.id=e.target_id WHERE e.type='MENTIONS' AND "
"t.name='addition' AND s.label='Section' AND s.name='First Test'"),
1);
th_cleanup(tmp);
PASS();
}
TEST(adoc_links_incremental) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dladoc_inc_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char repo[512];
char db[512];
char full_db[512];
snprintf(repo, sizeof(repo), "%s/repo", tmp);
snprintf(db, sizeof(db), "%s/inc.db", tmp);
snprintf(full_db, sizeof(full_db), "%s/full.db", tmp);
adoc_write_repo(repo, FIRST_TESTS);
ASSERT_EQ(dm_index(repo, db, NULL), 0);
/* the tagged region moves to another method: the page follows it */
th_write_file(TH_PATH(repo, "docs/src/test/java/example/FirstTests.java"),
"package example;\n"
"\n"
"class FirstTests {\n"
" // tag::user_guide[]\n"
" void helper() {}\n"
" // end::user_guide[]\n"
"\n"
" void addition() {\n"
" helper();\n"
" }\n"
"}\n");
ASSERT_EQ(dm_step(repo, db, full_db, "tag moved", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0);
/* a component attribute changes: every page re-resolves */
th_write_file(TH_PATH(repo, "docs/antora.yml"),
"name: guide\n"
"ext:\n"
" collector:\n"
" scan:\n"
" - dir: ./src/test\n"
" into: modules/ROOT/examples\n"
"asciidoc:\n"
" attributes:\n"
" javadoc-root: \"xref:attachment$api\"\n"
" Assertions: '{javadoc-root}/org.x/org/x/Calc.html[Calc]'\n");
ASSERT_EQ(dm_step(repo, db, full_db, "attribute change", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0);
th_cleanup(tmp);
PASS();
}
SUITE(doc_links_adoc) {
dm_ship_held_families();
RUN_TEST(adoc_scan_structure);
RUN_TEST(adoc_antora_scope_blob);
RUN_TEST(adoc_links_pipeline);
RUN_TEST(adoc_links_incremental);
cbm_doclink_test_reset_ships();
}
tests/test_doc_links_md.c
+863
View File
@@ -0,0 +1,863 @@
/*
* test_doc_links_md.c — Markdown documents -> MENTIONS edges.
*
* Extraction (doclink_md.c: which text is a reference, its family, line and
* section), the span classifier, the resolver (doc_links_md.c: paths, ranges,
* members, qualified names, the unresolved reasons, what is no reference at
* all), back navigation, and incremental == full. Pipeline tests index a real
* fixture through cbm_pipeline_run (helpers: test_doc_mentions_helpers.h).
*/
#include "../src/foundation/compat.h"
#include "test_framework.h"
#include "test_helpers.h"
#include "test_doc_mentions_helpers.h"
#include "cbm.h"
#include "doclink.h"
#include "pipeline/pipeline.h"
#include "pipeline/pipeline_internal.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* ── the fixture ─────────────────────────────────────────────────── */
/* docs/guide.md, line by line (the line numbers are asserted below). */
static const char MD_GUIDE[] =
"Intro names `pkg/config.go` before any heading.\n" /* 1 */
"\n" /* 2 */
"# Guide\n" /* 3 */
"\n" /* 4 */
"See [the config](../pkg/config.go) and [routing][r].\n" /* 5 */
"\n" /* 6 */
"[r]: ../app/routing.py\n" /* 7 */
"\n" /* 8 */
"## Paths\n" /* 9 */
"\n" /* 10 */
"Range `app/routing.py#L4-L5`, span `app/routing.py:1-9`, bare "
"app/applications.py.\n" /* 11 */
"Dir `app/middleware/`, route `/docs`, ![x](img/a.png) and "
"https://example.org/a/b.py.\n" /* 12 */
"\n" /* 13 */
"## Names\n" /* 14 */
"\n" /* 15 */
"Use `app.routing.APIRouter.add`, `app.middleware`, `pkg.Config.Name`, "
"`self.Name`.\n" /* 16 */
"Also `os.path.join`, `app.Nope`, `test_routing.only_in_tests`, `main.py`, "
"`../app/gone.py`.\n" /* 17 */
"\n" /* 18 */
"```\n" /* 19 */
"`app/fenced.py` inside a fence\n" /* 20 */
"```\n" /* 21 */
"<!-- `app/commented.py` in a comment -->\n" /* 22 */
"<code>app/html.py</code> in HTML, `cfg.Name` through an instance.\n"; /* 23 */
static void md_write_fixture(const char *tmp) {
th_write_file(TH_PATH(tmp, "docs/guide.md"), MD_GUIDE);
th_write_file(TH_PATH(tmp, "app/__init__.py"), "");
th_write_file(TH_PATH(tmp, "app/routing.py"), "class APIRouter:\n" /* 1 */
" prefix = ''\n" /* 2 */
"\n" /* 3 */
" def add(self, p):\n" /* 4 */
" return p\n" /* 5 */
"\n" /* 6 */
"\n" /* 7 */
"def helper():\n" /* 8 */
" return 1\n"); /* 9 */
th_write_file(TH_PATH(tmp, "app/applications.py"), "class App:\n"
" def middleware(self):\n"
" return None\n");
th_write_file(TH_PATH(tmp, "app/middleware/__init__.py"), "VALUE = 1\n");
th_write_file(TH_PATH(tmp, "tests/test_routing.py"), "def only_in_tests():\n"
" return 0\n");
th_write_file(TH_PATH(tmp, "lib/main.py"), "def run():\n"
" return None\n");
th_write_file(TH_PATH(tmp, "pkg/config.go"), "package pkg\n"
"\n"
"type Config struct {\n"
"\tName string\n"
"}\n"
"\n"
"func Util() {}\n");
}
/* ── extraction ──────────────────────────────────────────────────── */
static const CBMDocLink *md_token(const CBMFileResult *r, const char *raw) {
return dm_find_token(r, raw);
}
static bool md_ends_with(const char *s, const char *suffix) {
size_t a = s ? strlen(s) : 0;
size_t b = strlen(suffix);
return a >= b && strcmp(s + a - b, suffix) == 0;
}
TEST(doc_links_md_extract_tokens) {
CBMFileResult *r = dm_extract(MD_GUIDE, CBM_LANG_MARKDOWN, "docs/guide.md");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->doc_links.failed);
/* before the first heading: the file is the source */
const CBMDocLink *t = md_token(r, "pkg/config.go");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->line, 1);
ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_CODE_PATH);
ASSERT_TRUE((t->flags & CBM_DOCLINK_FLAG_FILE) != 0);
/* an inline link and a link reference definition, under "Guide" */
t = md_token(r, "../pkg/config.go");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->line, 5);
ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_LINK);
ASSERT_TRUE(md_ends_with(t->source_qn, "Guide"));
ASSERT_EQ(t->def_line, 3);
ASSERT_EQ(t->flags & CBM_DOCLINK_FLAG_FILE, 0);
t = md_token(r, "../app/routing.py");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->line, 7);
ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_LINK);
/* paths under "Paths" */
t = md_token(r, "app/routing.py#L4-L5");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->line, 11);
ASSERT_TRUE(md_ends_with(t->source_qn, "Paths"));
ASSERT_NOT_NULL(md_token(r, "app/routing.py:1-9"));
t = md_token(r, "app/applications.py");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_PATH);
t = md_token(r, "app/middleware/");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_CODE_PATH);
/* a token (the resolver calls it a route); one directory name alone is a
* bare path: in a held-out audit `doc/` was another project's as often as not */
t = md_token(r, "/docs");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_BARE_PATH);
/* an image, a URL: no tokens */
ASSERT_NULL(md_token(r, "img/a.png"));
ASSERT_NULL(md_token(r, "https://example.org/a/b.py"));
ASSERT_NULL(md_token(r, "a/b.py"));
/* qualified names under "Names" */
t = md_token(r, "app.routing.APIRouter.add");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_CODE_NAME);
ASSERT_EQ(t->line, 16);
ASSERT_TRUE(md_ends_with(t->source_qn, "Names"));
ASSERT_NOT_NULL(md_token(r, "pkg.Config.Name"));
ASSERT_NULL(md_token(r, "self.Name")); /* one identifier after `self.`: a bare name */
ASSERT_NOT_NULL(md_token(r, "os.path.join"));
/* a file name alone is its own family: in the held-out audit every wrong
* code path was one (`config.yaml`: the reader's own file) */
t = md_token(r, "main.py");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->syntax, CBM_DOCLINK_MD_BARE_PATH);
/* fenced code and HTML comments are not scanned; HTML <code> is */
ASSERT_NULL(md_token(r, "app/fenced.py"));
ASSERT_NULL(md_token(r, "app/commented.py"));
t = md_token(r, "app/html.py");
ASSERT_NOT_NULL(t);
ASSERT_EQ(t->line, 23);
cbm_free_result(r);
PASS();
}
/* Two headings with one title share one Section node, which owns only one of
* them: their references belong to the file, never to the other heading. */
TEST(doc_links_md_repeated_headings) {
const char *src = "# Changes\n" /* 1 */
"\n" /* 2 */
"## Fixes\n" /* 3 */
"\n" /* 4 */
"See `app/one.py`.\n" /* 5 */
"\n" /* 6 */
"## Fixes\n" /* 7 */
"\n" /* 8 */
"See `app/two.py`.\n" /* 9 */
"\n" /* 10 */
"## Unique\n" /* 11 */
"\n" /* 12 */
"See `app/three.py`.\n"; /* 13 */
CBMFileResult *r = dm_extract(src, CBM_LANG_MARKDOWN, "CHANGES.md");
ASSERT_NOT_NULL(r);
const CBMDocLink *one = md_token(r, "app/one.py");
const CBMDocLink *two = md_token(r, "app/two.py");
const CBMDocLink *three = md_token(r, "app/three.py");
ASSERT_NOT_NULL(one);
ASSERT_NOT_NULL(two);
ASSERT_NOT_NULL(three);
ASSERT_TRUE((one->flags & CBM_DOCLINK_FLAG_FILE) != 0);
ASSERT_TRUE((two->flags & CBM_DOCLINK_FLAG_FILE) != 0);
ASSERT_EQ(three->flags & CBM_DOCLINK_FLAG_FILE, 0);
ASSERT_TRUE(md_ends_with(three->source_qn, "Unique"));
cbm_free_result(r);
PASS();
}
typedef struct {
const char *text;
CBMDocLinkMdShape shape;
const char *path;
const char *member;
uint32_t first;
uint32_t last;
bool instance;
bool colon;
} md_case_t;
TEST(doc_links_md_classify_span) {
static const md_case_t cases[] = {
{"src/app/main.go", CBM_DOCLINK_MD_FILE, "src/app/main.go", NULL, 0, 0, false, false},
{"src/app/", CBM_DOCLINK_MD_DIR, "src/app/", NULL, 0, 0, false, false},
{"src/app/main.go#L3-L9", CBM_DOCLINK_MD_FILE, "src/app/main.go", NULL, 3, 9, false, false},
{"src/app/main.go:12", CBM_DOCLINK_MD_FILE, "src/app/main.go", NULL, 12, 12, false, false},
{"src/lib.rs::Type::run", CBM_DOCLINK_MD_FILE, "src/lib.rs", "Type::run", 0, 0, false,
false},
{"src\\win\\file.cs", CBM_DOCLINK_MD_FILE, "src/win/file.cs", NULL, 0, 0, false, false},
{"main.go:7-8", CBM_DOCLINK_MD_FILENAME, "main.go", NULL, 7, 8, false, false},
{"README.md", CBM_DOCLINK_MD_FILENAME, "README.md", NULL, 0, 0, false, false},
{"Makefile", CBM_DOCLINK_MD_FILENAME, "Makefile", NULL, 0, 0, false, false},
/* a built or shipped file is a file name, never a member (`OliveTin.exe`
* took the module stem of OliveTin.exe.manifest in a held-out audit) */
{"OliveTin.exe", CBM_DOCLINK_MD_FILENAME, "OliveTin.exe", NULL, 0, 0, false, false},
{"app.manifest", CBM_DOCLINK_MD_FILENAME, "app.manifest", NULL, 0, 0, false, false},
{"main.rs::run", CBM_DOCLINK_MD_FILENAME, "main.rs", "run", 0, 0, false, false},
{"fastapi.routing.APIRouter", CBM_DOCLINK_MD_QUALIFIED, "fastapi.routing.APIRouter", NULL,
0, 0, false, false},
{"crate::extract::Json", CBM_DOCLINK_MD_QUALIFIED, "extract.Json", NULL, 0, 0, false,
false},
{"Vec<u8>::new", CBM_DOCLINK_MD_QUALIFIED, "Vec.new", NULL, 0, 0, false, false},
{"self.app.state", CBM_DOCLINK_MD_QUALIFIED, "app.state", NULL, 0, 0, true, false},
{"pkg.module:attr", CBM_DOCLINK_MD_QUALIFIED, "pkg.module.attr", NULL, 0, 0, false, true},
{"Foo#bar", CBM_DOCLINK_MD_QUALIFIED, "Foo.bar", NULL, 0, 0, false, false},
{"App\\Http\\Kernel", CBM_DOCLINK_MD_QUALIFIED, "App.Http.Kernel", NULL, 0, 0, false,
false},
{"server.New()", CBM_DOCLINK_MD_QUALIFIED, "server.New", NULL, 0, 0, false, false},
/* no path and no qualified name */
{"foo", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"self.x", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"Node.js", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"github.com/x/y", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"https://x.org/a.py", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"$HOME/a.py", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"PATH=/usr/bin", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"--verbose", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"go run main.go", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"\"a/b.py\"", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"pkg/Type.Member", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"@scope/pkg", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
/* a traversal or a root names no directory (a held-out link: `../` in a
* warning about attacker-controlled paths) */
{"../", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"./", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
{"../..", CBM_DOCLINK_MD_NONE, NULL, NULL, 0, 0, false, false},
};
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
const md_case_t *c = &cases[i];
char buf[256];
CBMDocLinkMdPath p;
bool ok = cbm_doclink_md_classify_span(c->text, strlen(c->text), buf, sizeof(buf), &p);
if (ok != (c->shape != CBM_DOCLINK_MD_NONE) || p.shape != c->shape) {
printf(" case %s: shape %d, expected %d\n", c->text, (int)p.shape, (int)c->shape);
FAIL("classification differs");
}
if (!ok) {
continue;
}
if (strcmp(p.path, c->path) != 0 ||
(c->member ? !p.member || strcmp(p.member, c->member) != 0 : p.member != NULL) ||
p.first_line != c->first || p.last_line != c->last || p.instance != c->instance ||
p.colon != c->colon) {
printf(" case %s: path %s member %s lines %u-%u instance %d colon %d\n", c->text,
p.path, p.member ? p.member : "(none)", p.first_line, p.last_line, p.instance,
p.colon);
FAIL("classification differs");
}
}
/* a name never fits a buffer by being cut */
char tiny[8];
CBMDocLinkMdPath p;
ASSERT_FALSE(cbm_doclink_md_classify_span("src/app/main.go", 15, tiny, sizeof(tiny), &p));
PASS();
}
/* ── resolution ──────────────────────────────────────────────────── */
/* Properties of the MENTIONS edge from the section or file whose qualified
* name ends with `src` to the node whose qualified name ends with `tgt`. */
static int md_edge(const char *db, const char *src, const char *tgt, char *props, size_t cap) {
int n = 0;
dm_edge(db, src, tgt, props, cap, &n);
return n;
}
TEST(doc_links_md_resolve) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
md_write_fixture(tmp);
char db[512];
snprintf(db, sizeof(db), "%s/md.db", tmp);
ASSERT_EQ(dm_index(tmp, db, NULL), 0);
char props[512];
char reason[64];
char syntax[64];
/* a link and a link reference definition: their files */
ASSERT_EQ(md_edge(db, "Guide", "config.go.__file__", props, sizeof(props)), 1);
ASSERT_NOT_NULL(strstr(props, "\"via\":\"markdown\""));
ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"link\""));
ASSERT_NOT_NULL(strstr(props, "\"tier\":\"exact\""));
ASSERT_NOT_NULL(strstr(props, "\"line\":5"));
ASSERT_EQ(md_edge(db, "Guide", "routing.py.__file__", props, sizeof(props)), 1);
/* the reference before the first heading: from the File node */
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' "
"AND s.label = 'File' AND s.file_path = 'docs/guide.md' "
"AND t.file_path = 'pkg/config.go' AND t.label = 'File'"),
1);
/* a range held by one definition: that definition, the range on the edge */
ASSERT_EQ(md_edge(db, "Paths", "APIRouter.add", props, sizeof(props)), 1);
ASSERT_NOT_NULL(strstr(props, "\"target_lines\":[4,5]"));
ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"code_path\""));
/* a range across definitions: the file, with the range */
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' "
"AND s.qualified_name LIKE '%.Paths' AND t.label = 'File' "
"AND t.file_path = 'app/routing.py' "
"AND e.properties LIKE '%\"target_lines\":[1,9]%'"),
1);
/* a bare path in prose; a directory */
ASSERT_EQ(md_edge(db, "Paths", "applications.py.__file__", props, sizeof(props)), 1);
ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"path\""));
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' "
"AND s.qualified_name LIKE '%.Paths' AND t.label = 'Folder' "
"AND t.file_path = 'app/middleware'"),
1);
/* a route, an image, a URL: neither edges nor rows */
dm_row(db, "docs/guide.md", "/docs", reason, sizeof(reason), NULL, 0);
ASSERT_STR_EQ(reason, "");
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id = e.target_id "
"WHERE e.type = 'MENTIONS' AND t.label = 'Folder' "
"AND t.file_path = 'docs'"),
0);
dm_row(db, "docs/guide.md", "img/a.png", reason, sizeof(reason), NULL, 0);
ASSERT_STR_EQ(reason, "");
/* qualified names: a method, a field through its type */
ASSERT_EQ(md_edge(db, "Names", "routing.APIRouter.add", props, sizeof(props)), 1);
ASSERT_NOT_NULL(strstr(props, "\"syntax\":\"code_name\""));
ASSERT_EQ(md_edge(db, "Names", "Config.Name", props, sizeof(props)), 1);
/* `app.middleware`: the package as written, not the method App.middleware */
ASSERT_EQ(md_edge(db, "Names", "App.middleware", props, sizeof(props)), 0);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' "
"AND s.qualified_name LIKE '%.Names' "
"AND t.file_path = 'app/middleware/__init__.py'"),
1);
/* a field never through an instance */
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id = e.target_id "
"WHERE e.type = 'MENTIONS' AND t.label = 'Field' "
"AND e.properties LIKE '%\"line\":23%'"),
0);
/* unresolved, by reason */
dm_row(db, "docs/guide.md", "app.Nope", reason, sizeof(reason), syntax, sizeof(syntax));
ASSERT_STR_EQ(reason, "missing");
ASSERT_STR_EQ(syntax, "code_name");
dm_row(db, "docs/guide.md", "test_routing.only_in_tests", reason, sizeof(reason), NULL, 0);
ASSERT_STR_EQ(reason, "test_only_target");
dm_row(db, "docs/guide.md", "main.py", reason, sizeof(reason), syntax, sizeof(syntax));
ASSERT_STR_EQ(reason, "ambiguous"); /* lib/main.py, but not next to the document */
ASSERT_STR_EQ(syntax, "bare_path");
dm_row(db, "docs/guide.md", "../app/gone.py", reason, sizeof(reason), NULL, 0);
ASSERT_STR_EQ(reason, "missing");
dm_row(db, "docs/guide.md", "app/html.py", reason, sizeof(reason), NULL, 0);
ASSERT_STR_EQ(reason, "missing");
/* another project's code is no reference to this repository */
dm_row(db, "docs/guide.md", "os.path.join", reason, sizeof(reason), NULL, 0);
ASSERT_STR_EQ(reason, "");
/* back navigation: the sections that mention a method, one hop away */
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' "
"AND t.qualified_name LIKE '%.APIRouter.add' AND s.label = 'Section'"),
2);
th_cleanup(tmp);
PASS();
}
/* ── incremental == full ─────────────────────────────────────────── */
TEST(doc_links_md_incremental) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_inc_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char repo[512];
snprintf(repo, sizeof(repo), "%s/repo", tmp);
md_write_fixture(repo);
char db[512];
char full_db[512];
snprintf(db, sizeof(db), "%s/inc.db", tmp);
snprintf(full_db, sizeof(full_db), "%s/full.db", tmp);
ASSERT_EQ(dm_index(repo, db, NULL), 0);
/* a body edit of a linked code file: its linking document re-resolves */
th_write_file(TH_PATH(repo, "app/routing.py"), "class APIRouter:\n"
" prefix = ''\n"
"\n"
" def add(self, p):\n"
" return p + p\n"
"\n"
"\n"
"def helper():\n"
" return 2\n");
ASSERT_EQ(dm_step(repo, db, full_db, "code body edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR),
0);
/* the document itself changes: a new link, a reference gone */
char guide[sizeof(MD_GUIDE) + 64];
snprintf(guide, sizeof(guide), "%s\nSee also [helpers](../app/routing.py#L8-L9).\n", MD_GUIDE);
th_write_file(TH_PATH(repo, "docs/guide.md"), guide);
ASSERT_EQ(dm_step(repo, db, full_db, "document edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0);
/* a body edit that moves lines: the document bound by lines (#L8-L9, the
* helper until now) re-resolves, though no name changed */
th_write_file(TH_PATH(repo, "app/routing.py"), "# moved\n"
"# down\n"
"class APIRouter:\n"
" prefix = ''\n"
"\n"
" def add(self, p):\n"
" return p + p\n"
"\n"
"\n"
"def helper():\n"
" return 2\n");
ASSERT_EQ(dm_step(repo, db, full_db, "lines moved", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0);
/* the name a row calls missing is added: the run goes FULL and the row
* becomes an edge, exactly as a full index has it */
th_write_file(TH_PATH(repo, "app/nope.py"), "def Nope():\n"
" return None\n");
ASSERT_EQ(dm_step(repo, db, full_db, "missing name added", CBM_INCREMENTAL_ROUTE_FORCED_FULL),
0);
th_cleanup(tmp);
PASS();
}
/* ── ADRs ────────────────────────────────────────────────────────── */
static void md_write_adrs(const char *tmp, const char *first_status) {
char first[1024];
snprintf(first, sizeof(first),
"---\n"
"status: %s\n"
"date: 2024-03-05\n"
"deciders: Ana, Bo\n"
"---\n"
"# Use X for storage\n"
"\n"
"## Context and Problem Statement\n"
"\n"
"We need storage.\n"
"\n"
"## Decision Outcome\n"
"\n"
"We use X, see `app/routing.py`.\n"
"\n"
"Superseded by [ADR-0002](0002-use-y.md).\n",
first_status);
th_write_file(TH_PATH(tmp, "docs/adr/0001-use-x.md"), first);
th_write_file(TH_PATH(tmp, "docs/adr/0002-use-y.md"), "# 2. Use Y instead\n"
"\n"
"Date: 12 May 2024\n"
"\n"
"## Status\n"
"\n"
"Accepted\n"
"\n"
"Supersedes [ADR-0001](0001-use-x.md).\n"
"\n"
"## Context\n"
"\n"
"X was slow.\n"
"\n"
"## Decision\n"
"\n"
"We use Y.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0003-cache.md"),
"# Cache reads\n"
"\n"
"## Status\n"
"\n"
"Proposed\n"
"\n"
"- Replaces ADR-2\n"
"\n"
"## Decision\n"
"\n"
"The cache replaces the reader described in [ADR-1](0001-use-x.md).\n");
th_write_file(TH_PATH(tmp, "docs/adr/README.md"), "# Decisions\n\n## Status\n\nAccepted\n");
th_write_file(TH_PATH(tmp, "docs/adr/template.md"),
"# Title\n\n## Status\n\n{proposed | accepted}\n\n## Decision\n\nTBD\n");
}
/* Properties of the ADR node of `file`; "" when there is none. */
static void md_adr_props(const char *db, const char *file, char *out, size_t cap) {
out[0] = '\0';
sqlite3 *h = NULL;
if (sqlite3_open_v2(db, &h, SQLITE_OPEN_READONLY, NULL) == SQLITE_OK) {
sqlite3_stmt *st = NULL;
if (sqlite3_prepare_v2(h,
"SELECT name || ' ' || properties FROM nodes WHERE label = 'ADR' "
"AND file_path = ?1",
-1, &st, NULL) == SQLITE_OK) {
sqlite3_bind_text(st, 1, file, -1, SQLITE_TRANSIENT);
if (sqlite3_step(st) == SQLITE_ROW) {
snprintf(out, cap, "%s", (const char *)sqlite3_column_text(st, 0));
}
}
sqlite3_finalize(st);
}
sqlite3_close(h);
}
TEST(doc_links_md_adr) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmd_adr_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
md_write_fixture(tmp);
md_write_adrs(tmp, "superseded");
char db[512];
snprintf(db, sizeof(db), "%s/adr.db", tmp);
ASSERT_EQ(dm_index(tmp, db, NULL), 0);
char props[2048];
/* three records; README and template are none */
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label = 'ADR'"), 3);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id = e.target_id "
"WHERE e.type = 'DEFINES' AND t.label = 'ADR'"),
3);
/* facts: front matter */
md_adr_props(db, "docs/adr/0001-use-x.md", props, sizeof(props));
ASSERT_TRUE(strncmp(props, "ADR-1 ", strlen("ADR-1 ")) == 0);
ASSERT_NOT_NULL(strstr(props, "\"status\":\"superseded\""));
ASSERT_NOT_NULL(strstr(props, "\"date\":\"2024-03-05\""));
ASSERT_NOT_NULL(strstr(props, "\"deciders\":\"Ana, Bo\""));
ASSERT_NOT_NULL(strstr(props, "\"title\":\"Use X for storage\""));
ASSERT_NOT_NULL(strstr(props, "\"superseded_by\":\"0002-use-y.md\""));
ASSERT_NOT_NULL(strstr(props, "We use X")); /* the decision, searchable */
/* facts: header field date, Status section */
md_adr_props(db, "docs/adr/0002-use-y.md", props, sizeof(props));
ASSERT_TRUE(strncmp(props, "ADR-2 ", strlen("ADR-2 ")) == 0);
ASSERT_NOT_NULL(strstr(props, "\"status\":\"accepted\""));
ASSERT_NOT_NULL(strstr(props, "\"date\":\"2024-05-12\""));
md_adr_props(db, "docs/adr/README.md", props, sizeof(props));
ASSERT_STR_EQ(props, "");
/* SUPERSEDES: by link, and by id at a line head; never from prose that
* uses "replaces" as a verb, never from the superseded record */
const char *sup = "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"JOIN nodes t ON t.id = e.target_id WHERE e.type = 'SUPERSEDES' "
"AND s.file_path = '%s' AND t.file_path = '%s'";
char q[512];
snprintf(q, sizeof(q), sup, "docs/adr/0002-use-y.md", "docs/adr/0001-use-x.md");
ASSERT_EQ(dm_count(db, q), 1);
snprintf(q, sizeof(q), sup, "docs/adr/0003-cache.md", "docs/adr/0002-use-y.md");
ASSERT_EQ(dm_count(db, q), 1);
snprintf(q, sizeof(q), sup, "docs/adr/0003-cache.md", "docs/adr/0001-use-x.md");
ASSERT_EQ(dm_count(db, q), 0);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges WHERE type = 'SUPERSEDES'"), 2);
/* the record's sections still link to code */
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"JOIN nodes t ON t.id = e.target_id WHERE e.type = 'MENTIONS' "
"AND s.file_path = 'docs/adr/0001-use-x.md' "
"AND t.file_path = 'app/routing.py'"),
1);
/* incremental == full: a status edit, then a new record that a statement
* names by id */
char full_db[512];
snprintf(full_db, sizeof(full_db), "%s/adr-full.db", tmp);
md_write_adrs(tmp, "deprecated");
ASSERT_EQ(dm_step(tmp, db, full_db, "ADR status edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR),
0);
md_adr_props(db, "docs/adr/0001-use-x.md", props, sizeof(props));
ASSERT_NOT_NULL(strstr(props, "\"status\":\"deprecated\""));
th_write_file(TH_PATH(tmp, "docs/adr/0004-later.md"), "# Later\n\n## Status\n\nAccepted\n\n"
"Supersedes ADR-3.\n\n## Decision\n\n"
"Later.\n");
ASSERT_EQ(dm_step(tmp, db, full_db, "ADR added", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0);
snprintf(q, sizeof(q), sup, "docs/adr/0004-later.md", "docs/adr/0003-cache.md");
ASSERT_EQ(dm_count(db, q), 1);
th_cleanup(tmp);
PASS();
}
/* The production gate (doclink.h): families held back by the held-out audit
* write rows, not edges; the families that passed still write edges. */
static int md_supersedes_from(const char *db, const char *file, const char *target);
TEST(doc_links_md_ship_gate) {
for (size_t i = 0; i < sizeof(DM_HELD_FAMILIES) / sizeof(DM_HELD_FAMILIES[0]); i++) {
ASSERT_FALSE(cbm_doclink_syntax_ships(DM_HELD_FAMILIES[i]));
}
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_MD_LINK));
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_ROLE));
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_ADOC_INCLUDE));
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_LITERALINCLUDE));
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_CODE_PATH));
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_MD_CODE_PATH));
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_MD_SUPERSEDES));
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_OBJECT));
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmdgate_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
md_write_fixture(tmp);
/* a file name alone, next to its file: it resolves, and its tier holds it */
th_write_file(TH_PATH(tmp, "pkg/README.md"), "# Pkg\n\nSee `config.go`.\n");
/* an ADR's statement ships as SUPERSEDES; the same words in prose are held */
th_write_file(TH_PATH(tmp, "docs/adr/0001-base.md"),
"# Base\n\n## Status\n\nSuperseded\n\n## Decision\n\nWe use A.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0002-next.md"),
"# Next\n\n## Status\n\nAccepted\n\nSupersedes [ADR-0001](0001-base.md)\n\n"
"## Decision\n\nWe use B.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0003-last.md"),
"# Last\n\n## Status\n\nAccepted\n\n## Decision\n\n"
"This record also supersedes ADR-0001 for the cache.\n");
char db[512];
snprintf(db, sizeof(db), "%s/md.db", tmp);
ASSERT_EQ(dm_index(tmp, db, NULL), 0);
char props[512];
ASSERT_EQ(md_edge(db, "Guide", "config.go.__file__", props, sizeof(props)), 1);
ASSERT_EQ(md_edge(db, "Paths", "APIRouter.add", props, sizeof(props)), 1);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE "
"reason = 'below_bar_tier' AND syntax = 'code_path'"),
0);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE "
"reason = 'below_bar_tier' AND syntax = 'bare_path' "
"AND rel_path = 'pkg/README.md'"),
1);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges WHERE type = 'MENTIONS' AND "
"properties LIKE '%\"syntax\":\"bare_path\"%'"),
0);
ASSERT_EQ(md_supersedes_from(db, "0002-next.md", "0001-base.md"), 1);
ASSERT_EQ(md_supersedes_from(db, "0003-last.md", "%"), 0);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE "
"reason = 'below_bar_tier' AND syntax = 'supersedes_prose' "
"AND rel_path = 'docs/adr/0003-last.md'"),
1);
th_cleanup(tmp);
PASS();
}
/* A held-out finding: a spec template's combined field `Supersedes / Depends
* on: [Spec 2](...)` states no supersession (the record depended on the one it
* named); the plain statement next to it still does. */
TEST(doc_links_md_adr_label_choice) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dladrl_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
th_write_file(TH_PATH(tmp, "docs/adr/0001-base.md"),
"# Base\n\n## Status\n\nAccepted\n\n## Decision\n\nWe use A.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0002-next.md"),
"# Next\n\n## Status\n\nAccepted\n\n"
"Supersedes / Depends on: [ADR-0001](0001-base.md)\n\n"
"## Decision\n\nWe add B.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0003-last.md"), "# Last\n\n## Status\n\nAccepted\n\n"
"Supersedes [ADR-0001](0001-base.md).\n\n"
"## Decision\n\nWe use C.\n");
char db[512];
snprintf(db, sizeof(db), "%s/adr.db", tmp);
ASSERT_EQ(dm_index(tmp, db, NULL), 0);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"WHERE e.type = 'SUPERSEDES' AND s.file_path = 'docs/adr/0002-next.md'"),
0);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"WHERE e.type = 'SUPERSEDES' AND s.file_path = 'docs/adr/0003-last.md'"),
1);
th_cleanup(tmp);
PASS();
}
static int md_supersedes_from(const char *db, const char *file, const char *target) {
char sql[512];
snprintf(sql, sizeof(sql),
"SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"JOIN nodes t ON t.id = e.target_id WHERE e.type = 'SUPERSEDES' "
"AND s.file_path = 'docs/adr/%s' AND t.file_path LIKE 'docs/adr/%s'",
file, target);
return dm_count(db, sql);
}
static int md_supersedes_syntax(const char *db, const char *file, const char *syntax) {
char sql[512];
snprintf(sql, sizeof(sql),
"SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id = e.source_id "
"WHERE e.type = 'SUPERSEDES' AND s.file_path = 'docs/adr/%s' "
"AND e.properties LIKE '%%\"syntax\":\"%s\"%%'",
file, syntax);
return dm_count(db, sql);
}
/* Held-out findings: a field whose value is "none" goes on to name records it
* extends; a record reports another record's supersession (`ADR 0002
* supersedes ...`, a table cell `0002 (supersedes ...)`); the record after a
* statement's full stop is no target. The record's own number as the subject
* is its own statement. */
TEST(doc_links_md_adr_reported) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dladrr_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
th_write_file(TH_PATH(tmp, "docs/adr/0001-base.md"),
"# Base\n\n## Status\n\nAccepted\n\n## Decision\n\nWe use A.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0002-next.md"),
"# Next\n\n## Status\n\nAccepted\n\n## Decision\n\nWe use B.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0003-none.md"),
"# None\n\n- Status: Accepted\n"
"- Supersedes: none (it extends [ADR 0001](0001-base.md))\n\n"
"## Decision\n\nWe add C.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0004-na.md"),
"# NA\n\n- Status: Accepted\n- Supersedes: **N/A** -- refines ADR-0002\n\n"
"## Decision\n\nWe add D.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0005-report.md"),
"# Report\n\n## Status\n\nAccepted\n\n## Context\n\n"
"ADR 0002 supersedes ADR 0001's first decision.\n\n"
"| Row | Owner |\n|---|---|\n| retries | 0002 (supersedes ADR 0001's rule) |\n\n"
"[ADR 0002](0002-next.md) supersedes ADR 0001 too.\n\n"
"## Decision\n\nWe audit.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0006-own.md"),
"# Own\n\n## Status\n\nAccepted\n\n## Decision\n\n"
"ADR 0006 supersedes ADR 0001.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0007-stop.md"),
"# Stop\n\n## Status\n\nAccepted\n\n## Decision\n\n"
"> This record supersedes ADR-0001. ADR 0002 is an orthogonal decision.\n");
/* adr-tools' own form: the link text holds the record's number and title */
th_write_file(TH_PATH(tmp, "docs/adr/0008-tools.md"),
"# 8. Tools\n\n## Status\n\nAccepted\n\n"
"Supersedes [2. Next](0002-next.md)\n\n## Decision\n\nWe use E.\n");
/* statements in a field table and a bold field */
th_write_file(TH_PATH(tmp, "docs/adr/0009-table.md"),
"# Table\n\n| Field | Value |\n|---|---|\n| Status | Accepted |\n"
"| Supersedes | [ADR 0001](0001-base.md) |\n\n## Decision\n\nWe use F.\n");
th_write_file(TH_PATH(tmp, "docs/adr/0010-field.md"),
"# Field\n\n- Status: Accepted\n- **Supersedes:** ADR-0002\n\n"
"## Decision\n\nWe use G.\n");
/* a paragraph wrapped before the phrase: its subject is the line before */
th_write_file(
TH_PATH(tmp, "docs/adr/0011-wrap.md"),
"# Wrap\n\n## Status\n\nAccepted\n\n## Context\n\nThe sealed store of\n"
"[ADR 0002](0002-next.md)\nsupersedes ADR-0001 and leaves this record standing.\n\n"
"## Decision\n\nWe use I.\n");
/* field lines one under the other: each its own statement */
th_write_file(TH_PATH(tmp, "docs/adr/0012-fields.md"),
"# Fields\n\nStatus: Accepted\nSupersedes [ADR-0001](0001-base.md)\n\n"
"## Decision\n\nWe use J.\n");
/* a `Supersedes:` field right under running text is still a field */
th_write_file(TH_PATH(tmp, "docs/adr/0013-colon.md"),
"# Colon\n\n## Status\n\nAccepted after a long review of the store\n"
"Supersedes: [ADR 0002](0002-next.md)\n\n## Decision\n\nWe use K.\n");
char db[512];
snprintf(db, sizeof(db), "%s/adr.db", tmp);
ASSERT_EQ(dm_index(tmp, db, NULL), 0);
ASSERT_EQ(md_supersedes_from(db, "0003-none.md", "%"), 0);
ASSERT_EQ(md_supersedes_from(db, "0004-na.md", "%"), 0);
ASSERT_EQ(md_supersedes_from(db, "0005-report.md", "%"), 0);
ASSERT_EQ(md_supersedes_from(db, "0006-own.md", "0001-base.md"), 1);
ASSERT_EQ(md_supersedes_from(db, "0007-stop.md", "0001-base.md"), 1);
ASSERT_EQ(md_supersedes_from(db, "0007-stop.md", "0002-next.md"), 0);
ASSERT_EQ(md_supersedes_from(db, "0008-tools.md", "0002-next.md"), 1);
ASSERT_EQ(md_supersedes_from(db, "0009-table.md", "0001-base.md"), 1);
ASSERT_EQ(md_supersedes_from(db, "0010-field.md", "0002-next.md"), 1);
/* a statement opens its line or field and names a record directly; the
* same words in running prose are the held family supersedes_prose */
ASSERT_EQ(md_supersedes_syntax(db, "0006-own.md", "supersedes_prose"), 1);
ASSERT_EQ(md_supersedes_syntax(db, "0007-stop.md", "supersedes_prose"), 1);
ASSERT_EQ(md_supersedes_syntax(db, "0008-tools.md", "supersedes"), 1);
ASSERT_EQ(md_supersedes_syntax(db, "0009-table.md", "supersedes"), 1);
ASSERT_EQ(md_supersedes_syntax(db, "0010-field.md", "supersedes"), 1);
ASSERT_EQ(md_supersedes_syntax(db, "0011-wrap.md", "supersedes"), 0);
ASSERT_EQ(md_supersedes_syntax(db, "0011-wrap.md", "supersedes_prose"), 1);
ASSERT_EQ(md_supersedes_syntax(db, "0012-fields.md", "supersedes"), 1);
ASSERT_EQ(md_supersedes_syntax(db, "0013-colon.md", "supersedes"), 1);
th_cleanup(tmp);
PASS();
}
/* Held-out findings (AsciiDoc code names, which resolve like Markdown's): a
* `std::` name is the standard library's, not a specialization or header of
* this repository; a Java name written from its root package is absolute, not
* a relocated copy's tail; a type is named in its own case (`debug.log` is a
* log file, not Debug.log); a definition of another copy of the document's
* project (snapshots side by side) never answers the document. Each with a
* control that keeps its link. */
TEST(doc_links_md_name_scope_rules) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlmdscope_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
/* a compatibility header named like the standard one (Boost.Typeof's) */
th_write_file(TH_PATH(tmp, "lib/include/mylib/std/complex.hpp"),
"#include <complex>\nnamespace mylib { using cplx = int; }\n");
th_write_file(TH_PATH(tmp, "lib/include/mylib/widget.hpp"),
"namespace mylib {\nclass Widget {\n public:\n int size() const;\n};\n}\n");
th_write_file(
TH_PATH(tmp, "shaded/src/main/java/org/acme/shaded/com/google/protobuf/Message.java"),
"package org.acme.shaded.com.google.protobuf;\npublic class Message {}\n");
th_write_file(TH_PATH(tmp, "core/src/main/java/com/acme/Widget.java"),
"package com.acme;\npublic class Widget {}\n");
th_write_file(TH_PATH(tmp, "core/src/main/java/com/acme/Ref.java"),
"package com.acme;\npublic class Ref {\n public static class Debug {\n"
" public static void log(String s) {}\n }\n}\n");
th_write_file(
TH_PATH(tmp, "v1/src/main/java/org/acme/Store.java"),
"package org.acme;\npublic class Store {\n public static class Reader {}\n}\n");
th_write_file(TH_PATH(tmp, "v2/src/main/java/org/acme/Store.java"),
"package org.acme;\npublic interface Store {}\n");
th_write_file(
TH_PATH(tmp, "docs/guide.md"),
"# Keys\n\nKeys use `std::complex`, and `mylib::Widget` holds them.\n\n"
"# Wire\n\nProtobuf's `com.google.protobuf.Message`; ours is `com.acme.Widget`.\n\n"
"# Logs\n\nRead `debug.log`.\n\n"
"# Debug\n\nCall `Debug.log`.\n");
th_write_file(TH_PATH(tmp, "v2/docs/design.md"), "# Design\n\nThe `Store.Reader` reads.\n");
th_write_file(TH_PATH(tmp, "v1/docs/notes.md"), "# Notes\n\nThe `Store.Reader` reads.\n");
/* a configuration key that spells a Java package's tail; a Python package */
th_write_file(TH_PATH(tmp, "core/src/main/java/io/acme/snapshot/mode/SnapshotMode.java"),
"package io.acme.snapshot.mode;\npublic enum SnapshotMode { INITIAL }\n");
th_write_file(TH_PATH(tmp, "pyapp/__init__.py"), "");
th_write_file(TH_PATH(tmp, "pyapp/sub/__init__.py"), "from .core import run\n");
th_write_file(TH_PATH(tmp, "pyapp/sub/core.py"), "def run():\n return 1\n");
th_write_file(TH_PATH(tmp, "docs/config.md"),
"# Config\n\nSet `snapshot.mode` to `initial`.\n\n"
"# Py\n\nThe `pyapp.sub` package runs it.\n");
char db[512];
snprintf(db, sizeof(db), "%s/md.db", tmp);
ASSERT_EQ(dm_index(tmp, db, NULL), 0);
char props[512];
ASSERT_EQ(md_edge(db, "guide.Keys", "complex.hpp.__file__", props, sizeof(props)), 0);
ASSERT_EQ(md_edge(db, "guide.Keys", "mylib.Widget", props, sizeof(props)), 1);
ASSERT_EQ(md_edge(db, "guide.Wire", "protobuf.Message", props, sizeof(props)), 0);
ASSERT_EQ(md_edge(db, "guide.Wire", "com.acme.Widget", props, sizeof(props)), 1);
ASSERT_EQ(md_edge(db, "guide.Logs", "Debug.log", props, sizeof(props)), 0);
ASSERT_EQ(md_edge(db, "guide.Debug", "Debug.log", props, sizeof(props)), 1);
ASSERT_EQ(md_edge(db, "design.Design", "Store.Reader", props, sizeof(props)), 0);
ASSERT_EQ(md_edge(db, "notes.Notes", "Store.Reader", props, sizeof(props)), 1);
ASSERT_EQ(md_edge(db, "config.Config", "snapshot.mode", props, sizeof(props)), 0);
ASSERT_EQ(md_edge(db, "config.Py", "sub.__init__.py.__file__", props, sizeof(props)), 1);
th_cleanup(tmp);
PASS();
}
SUITE(doc_links_md) {
dm_ship_held_families();
RUN_TEST(doc_links_md_extract_tokens);
RUN_TEST(doc_links_md_repeated_headings);
RUN_TEST(doc_links_md_classify_span);
RUN_TEST(doc_links_md_resolve);
RUN_TEST(doc_links_md_incremental);
RUN_TEST(doc_links_md_adr);
RUN_TEST(doc_links_md_adr_label_choice);
RUN_TEST(doc_links_md_adr_reported);
RUN_TEST(doc_links_md_name_scope_rules);
cbm_doclink_test_reset_ships();
RUN_TEST(doc_links_md_ship_gate);
}
tests/test_doc_links_pdf.c
+1317
View File
File diff suppressed because it is too large Load Diff
tests/test_doc_links_rst.c
+572
View File
@@ -0,0 +1,572 @@
/*
* test_doc_links_rst.c — reStructuredText documents -> Sections and MENTIONS
* edges (doclink_rst.c, doclink_py.c, doc_links_rst.c).
*
* The line model (titles, literal blocks, directives, comments), the tokens
* and their resolution contexts, the Python scope blob (package re-exports,
* Sphinx conf.py), the resolver's rules as the field test measured them
* (forms and re-exports, a module never answers a member's role, bare members
* are never searched, a name written in full binds even test code while a
* search leaves it out, intersphinx and other projects are no rows), the C
* domain, include / literalinclude / kernel-doc, extlinks, inline literals,
* Sphinx's source_suffix in discovery, reST ADRs, incremental == full.
*/
#include "../src/foundation/compat.h"
#include "test_framework.h"
#include "test_helpers.h"
#include "test_doc_mentions_helpers.h"
#include "cbm.h"
#include "doclink.h"
#include "pipeline/pipeline.h"
#include "pipeline/pipeline_internal.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* ── the scanner ─────────────────────────────────────────────────── */
static const char *const RST_DOC =
"Intro names :class:`pkg.Top` before any title.\n" /* 1 */
"\n" /* 2 */
"=========\n" /* 3 */
" The API\n" /* 4 */
"=========\n" /* 5 */
"\n" /* 6 */
".. module:: pkg\n" /* 7 */
"\n" /* 8 */
"Use :class:`Model` and :meth:`~pkg.Model.save`::\n" /* 9 */
"\n" /* 10 */
" :class:`NotARole` in a literal block\n" /* 11 */
" Fake Title\n" /* 12 */
" ==========\n" /* 13 */
"\n" /* 14 */
".. code-block:: python\n" /* 15 */
"\n" /* 16 */
" x = :func:`nope`\n" /* 17 */
"\n" /* 18 */
".. class:: Model(name)\n" /* 19 */
"\n" /* 20 */
" .. method:: save(force=False)\n" /* 21 */
"\n" /* 22 */
" Saves; see :attr:`name`.\n" /* 23 */
"\n" /* 24 */
"Usage\n" /* 25 */
"-----\n" /* 26 */
"\n" /* 27 */
".. autofunction:: pkg.base.top\n" /* 28 */
"\n" /* 29 */
".. literalinclude:: ../pkg/base.py\n" /* 30 */
" :lines: 2-3\n" /* 31 */
"\n" /* 32 */
".. include:: ../pkg/snippet.py\n" /* 33 */
"\n" /* 34 */
".. include:: other.rst\n" /* 35 */
"\n" /* 36 */
".. kernel-doc:: drivers/x.c\n" /* 37 */
" :identifiers: foo bar\n" /* 38 */
"\n" /* 39 */
".. a comment with :class:`Hidden`\n" /* 40 */
"\n" /* 41 */
"Text ``see :class:`Hidden2` there`` and ``pkg/base.py``,\n" /* 42 */
"``pkg.base.top`` and :source:`a file <pkg/base.py>`.\n"; /* 43 */
static int rst_count(const CBMFileResult *r, int syntax) {
int n = 0;
for (int i = 0; i < r->doc_links.count; i++) {
n += r->doc_links.items[i].syntax == syntax;
}
return n;
}
static bool rst_raw_has(const CBMFileResult *r, const char *needle) {
for (int i = 0; i < r->doc_links.count; i++) {
if (strstr(r->doc_links.items[i].raw, needle)) {
return true;
}
}
return false;
}
static const CBMDefinition *rst_def(const CBMFileResult *r, const char *label, const char *name) {
for (int i = 0; i < r->defs.count; i++) {
const CBMDefinition *d = &r->defs.items[i];
if (d->label && strcmp(d->label, label) == 0 && d->name && strcmp(d->name, name) == 0) {
return d;
}
}
return NULL;
}
TEST(rst_scan_structure) {
CBMFileResult *r = dm_extract(RST_DOC, CBM_LANG_RST, "docs/api.rst");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->doc_links.failed);
/* titles: over+under and under only; a title shape in a literal block is none */
const CBMDefinition *api = rst_def(r, "Section", "The API");
const CBMDefinition *usage = rst_def(r, "Section", "Usage");
ASSERT_NOT_NULL(api);
ASSERT_NOT_NULL(usage);
ASSERT_NULL(rst_def(r, "Section", "Fake Title"));
ASSERT_EQ(api->start_line, 4);
ASSERT_EQ(api->end_line, 23);
ASSERT_EQ(usage->start_line, 25);
ASSERT_STR_EQ(api->qualified_name, "p.docs.api.The-API");
ASSERT_NOT_NULL(api->docstring);
ASSERT_NOT_NULL(strstr(api->docstring, "Use :class:`Model`"));
/* nothing from a literal block, a code directive, a comment or an inline literal */
ASSERT_FALSE(rst_raw_has(r, "NotARole"));
ASSERT_FALSE(rst_raw_has(r, "nope"));
ASSERT_FALSE(rst_raw_has(r, "Hidden"));
/* text before the first title belongs to the file */
const CBMDocLink *top = dm_find_token(r, ":class:`pkg.Top`\tclass\tpkg.Top\t\t\t");
ASSERT_NOT_NULL(top);
ASSERT_EQ(top->flags, CBM_DOCLINK_FLAG_FILE);
ASSERT_EQ(top->line, 1);
/* roles carry the module and class in force */
const CBMDocLink *model = dm_find_token(r, ":class:`Model`\tclass\tModel\tpkg\t\t");
ASSERT_NOT_NULL(model);
ASSERT_STR_EQ(model->source_qn, "p.docs.api.The-API");
ASSERT_EQ(model->line, 9);
ASSERT_NOT_NULL(dm_find_token(r, ":attr:`name`\tattr\tname\tpkg\tModel\t"));
/* object directives: the signature and Python's full name in its class */
ASSERT_NOT_NULL(dm_find_token(r, ".. class:: Model(name)\tclass\tModel(name)\tpkg\t\t\tModel"));
ASSERT_NOT_NULL(dm_find_token(
r, ".. method:: save(force=False)\tmethod\tsave(force=False)\tpkg\tModel\t\tModel.save"));
ASSERT_NOT_NULL(dm_find_token(r, ".. module:: pkg\tmodule\tpkg\tpkg\t\t\t"));
/* autodoc, literalinclude, an include of code (not of a document), kernel-doc names */
ASSERT_NOT_NULL(
dm_find_token(r, ".. autofunction:: pkg.base.top\tautofunction\tpkg.base.top\tpkg\t"));
ASSERT_NOT_NULL(
dm_find_token(r, ".. literalinclude:: ../pkg/base.py\t../pkg/base.py\t2-3\t\t"));
ASSERT_EQ(rst_count(r, CBM_DOCLINK_RST_INCLUDE), 1);
ASSERT_EQ(rst_count(r, CBM_DOCLINK_RST_KERNEL_DOC), 3);
ASSERT_NOT_NULL(dm_find_token(r, ".. kernel-doc:: drivers/x.c\tdrivers/x.c\tbar"));
/* inline literals naming a path or a qualified name; a titled extlink */
ASSERT_NOT_NULL(dm_find_token(r, "pkg/base.py"));
ASSERT_EQ(dm_find_token(r, "pkg/base.py")->syntax, CBM_DOCLINK_RST_CODE_PATH);
ASSERT_EQ(dm_find_token(r, "pkg.base.top")->syntax, CBM_DOCLINK_RST_CODE_NAME);
ASSERT_NOT_NULL(
dm_find_token(r, ":source:`a file <pkg/base.py>`\tsource\ta file <pkg/base.py>\tpkg\t\t"));
cbm_free_result(r);
PASS();
}
/* ── Python's scope ──────────────────────────────────────────────── */
TEST(rst_python_scope_blob) {
CBMFileResult *r = dm_extract("from .base import Model\n"
"from .sub.thing import helper as aid, other\n"
"from ..up import x\n"
"from os import *\n"
"from django.db import models\n"
"try:\n"
" from .opt import Nested\n"
"except ImportError:\n"
" pass\n",
CBM_LANG_PYTHON, "pkg/__init__.py");
ASSERT_NOT_NULL(r);
ASSERT_NOT_NULL(r->doc_scope);
ASSERT_STR_EQ(r->doc_scope, "py1\n"
"I\t1\tbase\tModel\tModel\n"
"I\t1\tsub.thing\thelper\taid\n"
"I\t1\tsub.thing\tother\tother\n"
"I\t2\tup\tx\tx\n"
"I\t0\tdjango.db\tmodels\tmodels\n");
ASSERT_EQ(r->doc_links.count, 0);
cbm_free_result(r);
r = dm_extract("project = 'x'\n"
"primary_domain = 'c'\n"
"intersphinx_mapping = {\n"
" 'python': ('https://docs.python.org/3', None),\n"
" \"sphinx\": (\"https://www.sphinx-doc.org/en/master/\", None),\n"
"}\n"
"extlinks = {\n"
" 'source': ('https://github.com/o/r/blob/main/%s', '%s'),\n"
" 'issue': ('https://github.com/o/r/issues/%s', '#%s'),\n"
"}\n"
"extlinks['tree'] = ('https://github.com/o/r/tree/main/%s', '%s')\n",
CBM_LANG_PYTHON, "docs/conf.py");
ASSERT_NOT_NULL(r);
ASSERT_STR_EQ(r->doc_scope, "py1\n"
"C\n"
"D\tc\n"
"S\tpython\n"
"S\tsphinx\n"
"X\tsource\n"
"X\ttree\n");
cbm_free_result(r);
r = dm_extract("from .x import y\n", CBM_LANG_PYTHON, "pkg/mod.py");
ASSERT_NOT_NULL(r);
ASSERT_NULL(r->doc_scope);
cbm_free_result(r);
PASS();
}
/* ── the pipeline ────────────────────────────────────────────────── */
static const char *const API_RST =
"Models\n" /* 1 */
"======\n" /* 2 */
"\n" /* 3 */
".. module:: pkg\n" /* 4 */
"\n" /* 5 */
"Use :class:`Model`, :func:`pkg.aid` and :meth:`save`.\n" /* 6 */
"Not :class:`python:dict`, :class:`otherlib.Thing`; gone\n" /* 7 */
":class:`pkg.Missing`.\n" /* 8 */
"\n" /* 9 */
".. currentmodule:: pkg.views\n" /* 10 */
"\n" /* 11 */
".. class:: View\n" /* 12 */
"\n" /* 13 */
" Calls :meth:`dispatch`.\n" /* 14 */
"\n" /* 15 */
"Testing\n" /* 16 */
"=======\n" /* 17 */
"\n" /* 18 */
"Run :class:`pkg.test.runner.Runner`, not :class:`Runner`.\n" /* 19 */
"\n" /* 20 */
"Code\n" /* 21 */
"====\n" /* 22 */
"\n" /* 23 */
"See :source:`pkg/base.py`, ``pkg.base.top`` and :c:func:`c_func`.\n" /* 24 */
"\n" /* 25 */
".. autofunction:: pkg.base.top\n" /* 26 */
"\n" /* 27 */
".. literalinclude:: ../pkg/base.py\n" /* 28 */
" :lines: 2-3\n" /* 29 */
"\n" /* 30 */
".. include:: /../pkg/sub/thing.py\n" /* 31 */
"\n" /* 32 */
".. kernel-doc:: csrc/lib.c\n" /* 33 */
" :identifiers: c_func\n"; /* 34 */
static void rst_write_repo(const char *repo, const char *init) {
th_write_file(TH_PATH(repo, "pkg/__init__.py"), init);
th_write_file(TH_PATH(repo, "pkg/base.py"), "class Model:\n"
" def save(self):\n"
" return None\n"
"\n"
"\n"
"def top():\n"
" return 1\n");
th_write_file(TH_PATH(repo, "pkg/other.py"), "class Model:\n"
" pass\n");
th_write_file(TH_PATH(repo, "pkg/sub/__init__.py"), "");
th_write_file(TH_PATH(repo, "pkg/sub/thing.py"), "def helper():\n"
" return 2\n");
th_write_file(TH_PATH(repo, "pkg/views.py"), "class View:\n"
" def dispatch(self):\n"
" return None\n");
th_write_file(TH_PATH(repo, "pkg/test/__init__.py"), "");
th_write_file(TH_PATH(repo, "pkg/test/runner.py"), "class Runner:\n"
" pass\n");
/* tests/ is no package: tests/dispatch's import name is `dispatch` */
th_write_file(TH_PATH(repo, "tests/dispatch/__init__.py"), "");
th_write_file(TH_PATH(repo, "tests/dispatch/test_x.py"), "def test_x():\n"
" return None\n");
th_write_file(TH_PATH(repo, "csrc/lib.c"), "int c_func(void) {\n"
" return 0;\n"
"}\n");
th_write_file(TH_PATH(repo, "docs/conf.py"),
"project = 'x'\n"
"source_suffix = {'.rst': 'restructuredtext', '.txt': 'restructuredtext'}\n"
"intersphinx_mapping = {\n"
" 'python': ('https://docs.python.org/3', None),\n"
"}\n"
"extlinks = {\n"
" 'source': ('https://github.com/o/r/blob/main/%s', '%s'),\n"
"}\n");
th_write_file(TH_PATH(repo, "docs/api.rst"), API_RST);
th_write_file(TH_PATH(repo, "docs/topics/guide.txt"), "Guide\n"
"=====\n"
"\n"
"The :class:`pkg.views.View`.\n");
th_write_file(TH_PATH(repo, "notes/readme.txt"), "Title\n"
"=====\n"
"\n"
"Plain :class:`pkg.views.View`.\n");
}
static const char *const INIT_PY = "from .base import Model\n"
"from .sub.thing import helper as aid\n";
static bool rst_edge(const char *db, const char *src, const char *tgt, const char *needle) {
char props[512];
int n = 0;
dm_edge(db, src, tgt, props, sizeof(props), &n);
return n == 1 && (!needle || strstr(props, needle));
}
static int rst_rows(const char *db, const char *raw_prefix, const char *reason) {
char sql[512];
snprintf(sql, sizeof(sql),
"SELECT COUNT(*) FROM doc_link_unresolved WHERE raw LIKE '%s%%' AND reason='%s'",
raw_prefix, reason);
return dm_count(db, sql);
}
TEST(rst_links_pipeline) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char repo[512];
char db[512];
snprintf(repo, sizeof(repo), "%s/repo", tmp);
snprintf(db, sizeof(db), "%s/g.db", tmp);
rst_write_repo(repo, INIT_PY);
ASSERT_EQ(dm_index(repo, db, NULL), 0);
/* a re-export through the module context: pkg.Model is pkg/base.py's */
ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.base.Model",
"{\"via\":\"rst\",\"syntax\":\"role\",\"tier\":\"exact\",\"line\":6"));
/* an aliased re-export: pkg.aid is helper */
ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.sub.thing.helper", "\"tier\":\"exact\""));
/* a bare member is never searched; intersphinx and another project: no rows */
ASSERT_EQ(dm_count(db,
"SELECT COUNT(*) FROM doc_link_unresolved WHERE raw LIKE ':meth:`save`%' "
"OR raw LIKE ':class:`python:dict`%' OR raw LIKE "
"':class:`otherlib.Thing`%'"),
0);
ASSERT_EQ(rst_rows(db, ":class:`pkg.Missing`", "missing"), 1);
/* a module never answers :meth: -- the class context does */
ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.views.View.dispatch", "\"tier\":\"exact\""));
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id=e.target_id WHERE "
"e.type='MENTIONS' AND t.file_path LIKE 'tests/%'"),
0);
/* the object directive declares View in its module */
ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.views.View", "\"syntax\":\"object\""));
/* a name written in full binds test code; a search for it does not */
ASSERT_TRUE(rst_edge(db, "docs.api.Testing", "pkg.test.runner.Runner", "\"tier\":\"exact\""));
ASSERT_EQ(rst_rows(db, ":class:`Runner`", "test_only_target"), 1);
/* extlink, inline literal, C domain, autodoc */
ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.py.__file__", "\"syntax\":\"role\""));
ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.top", "\"syntax\":\"code_name\""));
ASSERT_TRUE(rst_edge(db, "docs.api.Code", "csrc.lib.c_func", NULL));
/* literalinclude :lines: the innermost definition, the lines on the edge */
ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.Model.save", "\"target_lines\":[2,3]"));
/* include from the documentation set's directory; kernel-doc file */
ASSERT_TRUE(
rst_edge(db, "docs.api.Code", "pkg.sub.thing.py.__file__", "\"syntax\":\"include\""));
ASSERT_TRUE(rst_edge(db, "docs.api.Code", "csrc.lib.c.__file__", "\"syntax\":\"kernel_doc\""));
/* conf.py's source_suffix: .txt is reST in its documentation set only */
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label='Section' AND "
"file_path='docs/topics/guide.txt' AND name='Guide'"),
1);
ASSERT_TRUE(rst_edge(db, "docs.topics.guide.Guide", "pkg.views.View", "\"via\":\"rst\""));
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE file_path='notes/readme.txt'"), 0);
/* back navigation: the class knows the sections that name it */
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN "
"nodes t ON t.id=e.target_id WHERE e.type='MENTIONS' AND "
"t.name='View' AND t.label='Class' AND s.label='Section'"),
2);
th_cleanup(tmp);
PASS();
}
/* :c:member: names a field through its struct: the owner is the segment
* before the last dot (-> reads as .), however far the name is qualified. */
TEST(rst_c_member_owner) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_cm_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char repo[512];
char db[512];
snprintf(repo, sizeof(repo), "%s/repo", tmp);
snprintf(db, sizeof(db), "%s/g.db", tmp);
th_write_file(TH_PATH(repo, "csrc/shapes.c"), "struct point {\n"
" int x;\n"
" int y;\n"
"};\n"
"\n"
"struct box {\n"
" struct point corner;\n"
" int x;\n"
"};\n");
th_write_file(TH_PATH(repo, "docs/conf.py"), "project = 'x'\n");
th_write_file(TH_PATH(repo, "docs/api.rst"),
"Shapes\n"
"======\n"
"\n"
"Read :c:member:`point.x`, :c:member:`shapes.box.x` and\n"
":c:member:`struct point->y`; not :c:member:`box.corner.y`.\n");
ASSERT_EQ(dm_index(repo, db, NULL), 0);
/* the owner runs back to the start of the name */
ASSERT_TRUE(rst_edge(db, "docs.api.Shapes", "csrc.shapes.point.x", "\"syntax\":\"role\""));
/* the owner runs back to the dot before it */
ASSERT_TRUE(rst_edge(db, "docs.api.Shapes", "csrc.shapes.box.x", "\"syntax\":\"role\""));
ASSERT_TRUE(rst_edge(db, "docs.api.Shapes", "csrc.shapes.point.y", "\"syntax\":\"role\""));
/* corner is a field, not a struct with a y */
ASSERT_EQ(rst_rows(db, ":c:member:`box.corner.y`", "missing"), 1);
th_cleanup(tmp);
PASS();
}
TEST(rst_adr_record) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_adr_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char repo[512];
char db[512];
snprintf(repo, sizeof(repo), "%s/repo", tmp);
snprintf(db, sizeof(db), "%s/g.db", tmp);
th_write_file(TH_PATH(repo, "pkg/base.py"), "class Model:\n"
" pass\n");
th_write_file(TH_PATH(repo, "docs/decisions/0002-use-models.rst"), "2. Use models\n"
"=============\n"
"\n"
"Status\n"
"------\n"
"\n"
"Accepted\n"
"\n"
"Context\n"
"-------\n"
"\n"
"We store records.\n"
"\n"
"Decision\n"
"--------\n"
"\n"
"Use ``pkg.base.Model``.\n");
ASSERT_EQ(dm_index(repo, db, NULL), 0);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM nodes WHERE label='ADR' AND name='ADR-2' AND "
"json_extract(properties,'$.status')='accepted'"),
1);
ASSERT_TRUE(rst_edge(db, "docs.decisions.0002-use-models.Decision", "pkg.base.Model",
"\"syntax\":\"code_name\""));
th_cleanup(tmp);
PASS();
}
TEST(rst_links_incremental) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_inc_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char repo[512];
char db[512];
char full_db[512];
snprintf(repo, sizeof(repo), "%s/repo", tmp);
snprintf(db, sizeof(db), "%s/inc.db", tmp);
snprintf(full_db, sizeof(full_db), "%s/full.db", tmp);
rst_write_repo(repo, INIT_PY);
ASSERT_EQ(dm_index(repo, db, NULL), 0);
ASSERT_TRUE(rst_edge(db, "docs.api.Code", "pkg.base.Model.save", "\"target_lines\":[2,3]"));
/* a body edit moves the lines a literalinclude names: the document
* re-resolves with the code file (its names did not change) */
th_write_file(TH_PATH(repo, "pkg/base.py"), "# moved\n"
"# down\n"
"class Model:\n"
" def save(self):\n"
" return None\n"
"\n"
"\n"
"def top():\n"
" return 1\n");
ASSERT_EQ(dm_step(repo, db, full_db, "lines moved", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0);
ASSERT_FALSE(rst_edge(db, "docs.api.Code", "pkg.base.Model.save", "\"target_lines\":[2,3]"));
/* the document changes alone: it re-resolves through the stored re-exports */
th_write_file(TH_PATH(repo, "docs/api.rst"), "Models\n"
"======\n"
"\n"
".. module:: pkg\n"
"\n"
"Only :class:`Model` and :func:`aid`.\n");
ASSERT_EQ(dm_step(repo, db, full_db, "document edit", CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR), 0);
ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.base.Model", NULL));
/* the package re-exports another Model: the unchanged document follows */
th_write_file(TH_PATH(repo, "pkg/__init__.py"), "from .other import Model\n"
"from .sub.thing import helper as aid\n");
ASSERT_EQ(dm_step(repo, db, full_db, "re-export change", CBM_INCREMENTAL_ROUTE_FORCED_FULL), 0);
ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.other.Model", NULL));
th_cleanup(tmp);
PASS();
}
/* Two held-out findings (supplementary audit): `.. py:function:: autofit` on a
* page without a module bound the example script module examples/autofit.py
* (an object directive declares a definition, never a module), and
* `:lines: 5-` (line 5 to the file's end) was read as line 5 alone. */
TEST(rst_object_and_open_lines) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrsto_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char repo[512];
char db[512];
snprintf(repo, sizeof(repo), "%s/repo", tmp);
snprintf(db, sizeof(db), "%s/g.db", tmp);
th_write_file(TH_PATH(repo, "examples/autofit.py"), "print('autofit example')\n");
th_write_file(TH_PATH(repo, "examples/tail.py"), "import os\n" /* 1 */
"\n" /* 2 */
"\n" /* 3 */
"class Tail:\n" /* 4 */
" def a(self):\n" /* 5 */
" return 1\n" /* 6 */
"\n" /* 7 */
" def b(self):\n" /* 8 */
" return 2\n"); /* 9 */
th_write_file(TH_PATH(repo, "docs/conf.py"), "project = 'x'\n");
th_write_file(TH_PATH(repo, "docs/worksheet.rst"), "Worksheet\n"
"=========\n"
"\n"
".. py:function:: autofit()\n"
"\n"
" Fit the columns.\n"
"\n"
".. literalinclude:: ../examples/tail.py\n"
" :lines: 5-\n");
ASSERT_EQ(dm_index(repo, db, NULL), 0);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges e JOIN nodes t ON t.id=e.target_id WHERE "
"e.type='MENTIONS' AND t.file_path='examples/autofit.py'"),
0);
ASSERT_EQ(rst_rows(db, ".. py:function:: autofit", "missing"), 1);
/* lines 5 to 9 lie in class Tail; line 5 alone lies in method a */
ASSERT_TRUE(
rst_edge(db, "docs.worksheet.Worksheet", "examples.tail.Tail", "\"target_lines\":[5,9]"));
th_cleanup(tmp);
PASS();
}
/* The production gate (doclink.h): object directives ship since their
* held-out audit; include and kernel-doc are still held, so in the same
* documents they write rows, not edges. Runs after the suite's forced
* families are reset. */
TEST(rst_ship_gate) {
ASSERT_TRUE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_OBJECT));
ASSERT_FALSE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_INCLUDE));
ASSERT_FALSE(cbm_doclink_syntax_ships(CBM_DOCLINK_RST_KERNEL_DOC));
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_dlrst_gate_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char repo[512];
char db[512];
snprintf(repo, sizeof(repo), "%s/repo", tmp);
snprintf(db, sizeof(db), "%s/g.db", tmp);
rst_write_repo(repo, INIT_PY);
ASSERT_EQ(dm_index(repo, db, NULL), 0);
/* `.. class:: View` in pkg.views: an edge */
ASSERT_TRUE(rst_edge(db, "docs.api.Models", "pkg.views.View", "\"syntax\":\"object\""));
/* the include resolves, and its family holds it as a row */
ASSERT_FALSE(rst_edge(db, "docs.api.Code", "pkg.sub.thing.py.__file__", NULL));
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM doc_link_unresolved WHERE "
"reason = 'below_bar_tier' AND syntax = 'include'"),
1);
ASSERT_EQ(dm_count(db, "SELECT COUNT(*) FROM edges WHERE type = 'MENTIONS' AND "
"properties LIKE '%\"syntax\":\"kernel_doc\"%'"),
0);
th_cleanup(tmp);
PASS();
}
SUITE(doc_links_rst) {
dm_ship_held_families();
RUN_TEST(rst_scan_structure);
RUN_TEST(rst_python_scope_blob);
RUN_TEST(rst_links_pipeline);
RUN_TEST(rst_c_member_owner);
RUN_TEST(rst_object_and_open_lines);
RUN_TEST(rst_adr_record);
RUN_TEST(rst_links_incremental);
cbm_doclink_test_reset_ships();
RUN_TEST(rst_ship_gate);
}
tests/test_doc_mentions_helpers.h
+16
View File
@@ -425,4 +425,20 @@ static inline int dm_reason_lines(const char *db, const char *block) {
return n;
}
/* The document families held back from shipping (doclink.h: below the held-out
* audit's bar, or not yet audited). Their resolution is tested with them on,
* so it is the resolution an audit would ship; doc_links_md_ship_gate tests
* the production gate itself. */
static const int DM_HELD_FAMILIES[] = {
CBM_DOCLINK_MD_BARE_PATH, CBM_DOCLINK_MD_SUPERSEDES_PROSE, CBM_DOCLINK_RST_INCLUDE,
CBM_DOCLINK_RST_KERNEL_DOC, CBM_DOCLINK_ADOC_ATTRIBUTE, CBM_DOCLINK_ADOC_CODE_PATH,
CBM_DOCLINK_ADOC_CODE_NAME,
};
static inline void dm_ship_held_families(void) {
for (size_t i = 0; i < sizeof(DM_HELD_FAMILIES) / sizeof(DM_HELD_FAMILIES[0]); i++) {
cbm_doclink_test_set_ships(DM_HELD_FAMILIES[i], true);
}
}
#endif /* TEST_DOC_MENTIONS_HELPERS_H */
tests/test_extraction.c
+75
View File
@@ -586,6 +586,48 @@ TEST(java_class) {
PASS();
}
/* A class field is one Field. Java, C# and Solidity list the field
* declaration as a variable kind too, and the class-body variable pass minted
* a second node: a Variable with the module's qualified name (Query.filters
* became `demo.query.filters`, the name of a package a documentation code
* span then bound). Languages whose class variables are no Field keep them. */
TEST(class_field_is_one_field_not_also_a_variable) {
CBMFileResult *r = extract("package demo.query;\npublic class Query {\n"
" private Object filters;\n"
" private static final String PROP = \"v\";\n}\n",
CBM_LANG_JAVA, "t", "Query.java");
ASSERT_NOT_NULL(r);
ASSERT_EQ(count_defs_named(r, "Field", "filters"), 1);
ASSERT_EQ(count_defs_named(r, "Variable", "filters"), 0);
/* an initialized field is named by its declarator's name, not its text
* (`PROP = "v"` was the Field's name, so only the duplicate was findable) */
ASSERT_EQ(count_defs_named(r, "Field", "PROP"), 1);
ASSERT_EQ(count_defs_named(r, "Variable", "PROP"), 0);
cbm_free_result(r);
r = extract("namespace Demo {\n public class Q {\n private int count;\n }\n}\n",
CBM_LANG_CSHARP, "t", "Q.cs");
ASSERT_NOT_NULL(r);
ASSERT_EQ(count_defs_named(r, "Field", "count"), 1);
ASSERT_EQ(count_defs_named(r, "Variable", "count"), 0);
cbm_free_result(r);
r = extract("pragma solidity ^0.8.0;\ncontract Q {\n uint256 public total;\n}\n",
CBM_LANG_SOLIDITY, "t", "Q.sol");
ASSERT_NOT_NULL(r);
ASSERT_EQ(count_defs_named(r, "Field", "total"), 1);
ASSERT_EQ(count_defs_named(r, "Variable", "total"), 0);
cbm_free_result(r);
/* no Field for these: their class variables stay */
r = extract("class Q {\n var count: Int = 0\n}\n", CBM_LANG_SWIFT, "t", "Q.swift");
ASSERT_NOT_NULL(r);
ASSERT_EQ(count_defs_named(r, "Variable", "count"), 1);
cbm_free_result(r);
r = extract("extends Node\nclass Inner:\n var x = 1\n", CBM_LANG_GDSCRIPT, "t", "q.gd");
ASSERT_NOT_NULL(r);
ASSERT_EQ(count_defs_named(r, "Variable", "x"), 1);
cbm_free_result(r);
PASS();
}
TEST(java_method) {
CBMFileResult *r = extract(
"public class Svc { public void doWork() {} public int compute(int x) { return x; } }",
@@ -7430,6 +7472,37 @@ static const CBMDefinition *c1_def_qn(CBMFileResult *r, const char *qn) {
return NULL;
}
static bool qn_ends_with(const CBMDefinition *d, const char *tail) {
size_t n = d && d->qualified_name ? strlen(d->qualified_name) : 0;
size_t t = strlen(tail);
return n >= t && strcmp(d->qualified_name + n - t, tail) == 0;
}
/* A field and a same-named method of one class body would share one qualified
* name (one node per QN): the field is fenced `<Owner>.<name>#field` and the
* method keeps the plain name. A field without a namesake keeps its name, and
* so does a nested class's field named like a method of the outer class. */
TEST(field_named_like_method_is_fenced) {
CBMFileResult *r = extract("package demo;\npublic class Starter {\n"
" private String executable;\n"
" private String dir;\n"
" public Starter executable(String p) { return this; }\n"
" public void size() {}\n"
" static class Inner { int size; }\n}\n",
CBM_LANG_JAVA, "t", "Starter.java");
ASSERT_NOT_NULL(r);
const CBMDefinition *field = c1_def(r, "Field", "executable");
const CBMDefinition *method = c1_def(r, "Method", "executable");
ASSERT_NOT_NULL(field);
ASSERT_NOT_NULL(method);
ASSERT_TRUE(qn_ends_with(field, "Starter.executable#field"));
ASSERT_TRUE(qn_ends_with(method, "Starter.executable"));
ASSERT_TRUE(qn_ends_with(c1_def(r, "Field", "dir"), "Starter.dir"));
ASSERT_TRUE(qn_ends_with(c1_def(r, "Field", "size"), "Inner.size"));
cbm_free_result(r);
PASS();
}
/* The def with this label and name that starts on `line`. */
static const CBMDefinition *c1_def_at(CBMFileResult *r, const char *label, const char *name,
int line) {
@@ -10398,6 +10471,8 @@ SUITE(extraction) {
/* OOP */
RUN_TEST(java_class);
RUN_TEST(java_method);
RUN_TEST(class_field_is_one_field_not_also_a_variable);
RUN_TEST(field_named_like_method_is_fenced);
RUN_TEST(java_interface);
RUN_TEST(java_interface_no_duplicate_function_issue1234);
RUN_TEST(java_enum_dedup_preserves_calls_issue1234);
tests/test_grammar_labels.c
+1 -1
View File
@@ -195,7 +195,7 @@ static const LabelGolden LABEL_GOLDENS[] = {
{"css", "Module:1"},
{"scss", "Module:1,Variable:1"},
{"markdown", "Module:1,Section:1"},
{"rst", "Module:1"},
{"rst", "Module:1,Section:1"},
{"dockerfile", "Module:1"},
{"makefile", "Function:1,Module:1"},
{"cmake", "Function:1,Module:1"},
tests/test_grammar_probe_g.c
+8 -7
View File
@@ -399,12 +399,13 @@ TEST(probe_markdown_no_class_variable) {
/* ══════════════════════════════════════════════════════════════════
* GROUP 6 — RST (.rst)
*
* RST golden histogram: Module:1 (pure-data).
* reStructuredText headings are NOT extracted as Section nodes (unlike Markdown).
* RST golden histogram: Module:1,Section:1.
* reStructuredText titles are Section nodes, like Markdown headings
* (doclink_rst.c).
* ══════════════════════════════════════════════════════════════════ */
/* RST: document with sections → Module:1, no Section nodes. */
TEST(probe_rst_module_only) {
/* RST: a document with two titles → Module:1 and a Section per title. */
TEST(probe_rst_sections) {
GpgMetrics m = gpg_metrics("docs.rst", "Project Title\n"
"=============\n"
"\n"
@@ -415,9 +416,9 @@ TEST(probe_rst_module_only) {
"\n"
"Run ``pip install mypackage``.\n");
ASSERT_TRUE(m.ok);
/* GREEN: RST produces only a Module node (no Section nodes extracted). */
/* GREEN: a Module node and the two titles' Section nodes. */
ASSERT_TRUE(m.modules == 1);
ASSERT_TRUE(m.sections == 0);
ASSERT_TRUE(m.sections == 2);
PASS();
}
@@ -1063,7 +1064,7 @@ SUITE(grammar_probe_g) {
RUN_TEST(probe_markdown_no_class_variable);
/* RST */
RUN_TEST(probe_rst_module_only);
RUN_TEST(probe_rst_sections);
RUN_TEST(probe_rst_no_crash);
/* JSON */
tests/test_main.c
+8
View File
@@ -1821,6 +1821,10 @@ extern void suite_doclinks(void);
extern void suite_doc_mentions(void);
extern void suite_doc_mentions_cost(void);
extern void suite_doc_mentions_msbuild(void);
extern void suite_doc_links_md(void);
extern void suite_doc_links_pdf(void);
extern void suite_doc_links_rst(void);
extern void suite_doc_links_adoc(void);
extern void suite_infrascan(void);
extern void suite_cli(void);
extern void suite_agent_clients(void);
@@ -2288,6 +2292,10 @@ int main(int argc, char **argv) {
RUN_SELECTED_SUITE(doc_mentions);
RUN_SELECTED_SUITE(doc_mentions_cost);
RUN_SELECTED_SUITE(doc_mentions_msbuild);
RUN_SELECTED_SUITE(doc_links_md);
RUN_SELECTED_SUITE(doc_links_pdf);
RUN_SELECTED_SUITE(doc_links_rst);
RUN_SELECTED_SUITE(doc_links_adoc);
/* Infrastructure scanning */
RUN_SELECTED_SUITE(infrascan);
tests/test_pipeline.c
+69
View File
@@ -1690,6 +1690,74 @@ static int fixture_node_count(cbm_store_t *store, const char *project, const cha
return found;
}
/* Edges of `edge_type` from the node named `source_name` to the node named
* `target_name` that carries `target_label`. */
static int labeled_edge_count(cbm_store_t *s, const char *project, const char *edge_type,
const char *source_name, const char *target_name,
const char *target_label) {
cbm_edge_t *edges = NULL;
int edge_count = 0;
if (cbm_store_find_edges_by_type(s, project, edge_type, &edges, &edge_count) != CBM_STORE_OK) {
return -1;
}
int matches = 0;
for (int i = 0; i < edge_count; i++) {
cbm_node_t source = {0};
cbm_node_t target = {0};
int source_ok = cbm_store_find_node_by_id(s, edges[i].source_id, &source) == CBM_STORE_OK;
int target_ok = cbm_store_find_node_by_id(s, edges[i].target_id, &target) == CBM_STORE_OK;
if (source_ok && target_ok && source.name && target.name && target.label &&
strcmp(source.name, source_name) == 0 && strcmp(target.name, target_name) == 0 &&
strcmp(target.label, target_label) == 0) {
matches++;
}
cbm_node_free_fields(&source);
cbm_node_free_fields(&target);
}
if (edges) {
cbm_store_free_edges(edges, edge_count);
}
return matches;
}
/* A Java field and a same-named method of one class keep their own nodes (the
* field is fenced `<Owner>.<name>#field`), and a write or a read of the field
* reaches the field, never the method; the method keeps its calls. The setter
* writing its own namesake was a self-edge before, so it was dropped. */
TEST(pipeline_field_named_like_method_keeps_its_reads_and_writes) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "/tmp/cbm_fieldfence_XXXXXX");
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
th_write_file(TH_PATH(tmp, "src/demo/Starter.java"),
"package demo;\n\npublic class Starter {\n"
" private String executable;\n\n"
" public Starter executable(String path) {\n"
" this.executable = path;\n"
" return this;\n }\n\n"
" public String show() {\n return executable;\n }\n\n"
" public static void run() {\n"
" new Starter().executable(\"x\").show();\n }\n}\n");
char db_path[512];
snprintf(db_path, sizeof(db_path), "%s/fence.db", tmp);
cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
const char *project = cbm_pipeline_project_name(p);
cbm_store_t *store = cbm_store_open_path(db_path);
ASSERT_NOT_NULL(store);
ASSERT_EQ(named_node_count(store, project, "executable"), 2);
ASSERT_EQ(labeled_edge_count(store, project, "WRITES", "executable", "executable", "Field"), 1);
ASSERT_EQ(labeled_edge_count(store, project, "WRITES", "executable", "executable", "Method"),
0);
ASSERT_EQ(labeled_edge_count(store, project, "USAGE", "show", "executable", "Field"), 1);
ASSERT_EQ(labeled_edge_count(store, project, "USAGE", "show", "executable", "Method"), 0);
ASSERT_EQ(labeled_edge_count(store, project, "CALLS", "run", "executable", "Method"), 1);
cbm_store_close(store);
cbm_pipeline_free(p);
th_cleanup(tmp);
PASS();
}
typedef struct {
int run_rc;
bool store_opened;
@@ -19314,6 +19382,7 @@ SUITE(pipeline) {
RUN_TEST(pipeline_nix_scoped_binding_calls_resolve);
RUN_TEST(pipeline_hcl_block_reference_resolves_to_its_block);
RUN_TEST(pipeline_incremental_preserves_cross_file_calls);
RUN_TEST(pipeline_field_named_like_method_keeps_its_reads_and_writes);
RUN_TEST(pipeline_objectscript_export_preserves_calls_sequential_parallel);
RUN_TEST(pipeline_objectscript_export_incremental_matches_full_relationships);
RUN_TEST(pipeline_js_route_not_bound_to_python_builtin_sequential);
tests/test_registry.c
+22
View File
@@ -213,6 +213,27 @@ TEST(registry_label_of) {
PASS();
}
/* A read, write or value reference that resolved to a Method goes to the
* method's fenced `#field` twin when the class has one; nothing else moves. */
TEST(registry_value_target_field_twin) {
cbm_registry_t *r = cbm_registry_new();
cbm_registry_add(r, "count", "p.C.count", "Method");
cbm_registry_add(r, "count", "p.C.count#field", "Field");
cbm_registry_add(r, "size", "p.C.size", "Method");
cbm_registry_add(r, "name", "p.C.name", "Field");
ASSERT_STR_EQ(cbm_registry_value_target(r, "p.C.count"), "p.C.count#field");
ASSERT_STR_EQ(cbm_registry_value_target(r, "p.C.size"), "p.C.size");
ASSERT_STR_EQ(cbm_registry_value_target(r, "p.C.name"), "p.C.name");
ASSERT_STR_EQ(cbm_registry_value_target(r, "p.C.none"), "p.C.none");
/* the fenced field is still found by its plain name */
const char **qns = NULL;
int n = 0;
ASSERT_EQ(cbm_registry_find_by_name(r, "count", &qns, &n), 0);
ASSERT_EQ(n, 2);
cbm_registry_free(r);
PASS();
}
TEST(registry_find_by_name) {
cbm_registry_t *r = cbm_registry_new();
cbm_registry_add(r, "main", "proj.cmd.main", "Function");
@@ -1569,6 +1590,7 @@ SUITE(registry) {
RUN_TEST(registry_free_null);
RUN_TEST(registry_add_and_exists);
RUN_TEST(registry_label_of);
RUN_TEST(registry_value_target_field_twin);
RUN_TEST(registry_find_by_name);
RUN_TEST(registry_no_duplicates);
/* Resolution */