Merge remote-tracking branch 'origin/main' into shared/core-land

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>
This commit is contained in:
Martin Vogel
2026-11-19 01:37:18 +01:00
23 changed files with 1282 additions and 90 deletions
+85 -12
View File
@@ -28,6 +28,7 @@
#include "cpp_branch_views.h"
#include "foundation/compat.h"
#include "foundation/compat_fs.h" // cbm_fopen — crash-supervisor per-file marker write
#include "foundation/limits.h" // cbm_max_file_bytes — headers read by the #include lookup
#include "foundation/hash_table.h" // CBMHashTable — crash-supervisor quarantine set
#include "foundation/str_util.h" // cbm_json_escape — variants file_path
#include "tree_sitter/api.h" // TSParser, TSNode, TSTree, TSInput, TSLanguage, TSPoint, TSParseOptions, TSParseState
@@ -3823,14 +3824,81 @@ static void cbm_swift_type_imports(CBMFileResult *result, CBMLanguage language)
}
}
static CBMFileResult *extract_file_ex_body(const char *source, int source_len, CBMLanguage language,
const char *project, const char *rel_path,
int64_t timeout_micros, const char **extra_defines,
const char **include_paths,
const CBMMacroTable *macro_table,
const CBMReturnTypeTable *return_type_table,
CBMArena *scratch,
const cbm_test_declarations_t *test_declarations) {
/* ── #include lookup of the preprocessed pass (#2993) ──────────────
* Quoted #include names of the C-family second pass resolve against the
* indexed repository: preprocessor.cpp hands over a normalised, root-relative
* name, and the header is read only when it is one of the files the index run
* covers (CBMIncludeScope.is_indexed; discovery has already applied its rules:
* no symlinks or junctions, ignore files, skipped directories), within the
* per-file size cap. Every file the expansion depends on is therefore in the
* run's manifest. Paths are UTF-8, opened through cbm_fopen. */
static char *include_tree_read(const void *userdata, const char *rel_path, size_t *out_len) {
const CBMIncludeScope *scope = (const CBMIncludeScope *)userdata;
if (!rel_path[0] || !scope->is_indexed || !scope->is_indexed(scope->userdata, rel_path)) {
return NULL;
}
size_t root_len = strlen(scope->root);
bool sep =
root_len > 0 && scope->root[root_len - 1] != '/' && scope->root[root_len - 1] != '\\';
char path[CBM_SZ_4K];
int written = snprintf(path, sizeof(path), "%s%s%s", scope->root, sep ? "/" : "", rel_path);
if (written < 0 || (size_t)written >= sizeof(path)) {
return NULL;
}
FILE *f = cbm_fopen(path, "rb");
if (!f) {
return NULL;
}
long size = fseek(f, 0, SEEK_END) == 0 ? ftell(f) : -1;
if (size < 0 || size > cbm_max_file_bytes() || fseek(f, 0, SEEK_SET) != 0) {
(void)fclose(f);
return NULL;
}
char *data = (char *)cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)size + 1);
size_t got = data && size > 0 ? fread(data, 1, (size_t)size, f) : 0;
(void)fclose(f);
if (data) {
data[got] = '\0';
*out_len = got;
}
return data;
}
static void include_tree_release(const void *userdata, char *data) {
(void)userdata;
cbm_free(CBM_MEM_CLASS_EXTRACT, data);
}
/* result->included_files from what the preprocessed pass's lookup read. An
* allocation failure marks the result as failed: a missing list would let an
* incremental refresh skip this file after one of its headers changed. */
static void record_included_files(CBMArena *a, CBMFileResult *result,
const CBMPreprocessedSource *pp) {
if (!pp || pp->included_count <= 0) {
return;
}
const char **list =
(const char **)cbm_arena_alloc(a, ((size_t)pp->included_count + 1) * sizeof(char *));
for (int i = 0; list && i < pp->included_count; i++) {
list[i] = cbm_arena_strdup(a, pp->included_files[i]);
if (!list[i]) {
list = NULL;
}
}
if (!list) {
result->has_error = true;
return;
}
list[pp->included_count] = NULL;
result->included_files = list;
}
static CBMFileResult *extract_file_ex_body(
const char *source, int source_len, CBMLanguage language, const char *project,
const char *rel_path, int64_t timeout_micros, const char **extra_defines,
const char **include_paths, const CBMMacroTable *macro_table,
const CBMReturnTypeTable *return_type_table, CBMArena *scratch,
const cbm_test_declarations_t *test_declarations, const CBMIncludeScope *include_scope) {
// Allocate result on heap (arena inside for all string data)
CBMFileResult *result = cbm_result_alloc();
if (!result) {
@@ -4223,8 +4291,12 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
char export_cands[CBM_EXPORT_MACRO_MAX][CBM_EXPORT_MACRO_NAME_MAX];
int export_cand_count =
cbm_export_macro_candidates(source, source_len, export_cands, CBM_EXPORT_MACRO_MAX);
CBMIncludeReader include_reader = {include_tree_read, include_tree_release, include_scope};
bool include_lookup = include_scope && include_scope->root && include_scope->root[0];
CBMPreprocessedSource *preprocessed = cbm_preprocess_with_map(
source, source_len, rel_path, extra_defines, include_paths, language != CBM_LANG_C);
source, source_len, rel_path, extra_defines, include_paths, language != CBM_LANG_C,
include_lookup ? &include_reader : NULL);
record_included_files(a, result, preprocessed);
if (preprocessed && preprocessed->source) {
char *expanded = preprocessed->source;
int expanded_len = (int)strlen(expanded);
@@ -4893,14 +4965,15 @@ CBMFileResult *cbm_extract_file_ex(const char *source, int source_len, CBMLangua
const CBMReturnTypeTable *return_type_table) {
return cbm_extract_file_ex_with_tests(source, source_len, language, project, rel_path,
timeout_micros, extra_defines, include_paths, macro_table,
return_type_table, NULL);
return_type_table, NULL, NULL);
}
CBMFileResult *cbm_extract_file_ex_with_tests(
const char *source, int source_len, CBMLanguage language, const char *project,
const char *rel_path, int64_t timeout_micros, const char **extra_defines,
const char **include_paths, const CBMMacroTable *macro_table,
const CBMReturnTypeTable *return_type_table, const cbm_test_declarations_t *test_declarations) {
const CBMReturnTypeTable *return_type_table, const cbm_test_declarations_t *test_declarations,
const CBMIncludeScope *include_scope) {
CBMArena scratch;
if (tl_scratch_live && tl_scratch_slot) {
scratch = *tl_scratch_slot;
@@ -4911,7 +4984,7 @@ CBMFileResult *cbm_extract_file_ex_with_tests(
}
CBMFileResult *result = extract_file_ex_body(
source, source_len, language, project, rel_path, timeout_micros, extra_defines,
include_paths, macro_table, return_type_table, &scratch, test_declarations);
include_paths, macro_table, return_type_table, &scratch, test_declarations, include_scope);
/* Even early parse/quarantine exits publish conservative Python metadata.
* No tree is reparsed, and existing best-effort extraction flags stay intact. */
if (result && language == CBM_LANG_PYTHON &&
+21 -2
View File
@@ -967,6 +967,11 @@ typedef struct CBMFileResult {
const char **constants; // NULL-terminated (NULL if none)
const char **global_vars; // NULL-terminated (NULL if none)
const char **macros; // NULL-terminated, C/C++ only (NULL if none)
/* C, C++, CUDA: the repository files the preprocessed pass's #include
* lookup read (repository-relative, sorted, NULL-terminated; NULL if none).
* The file's graph depends on their bytes as well as its own, so an
* incremental refresh re-extracts the file when one of them changes. */
const char **included_files;
bool has_error;
const char *error_msg;
@@ -1315,15 +1320,29 @@ CBMFileResult *cbm_extract_file_ex(
const CBMReturnTypeTable *return_type_table // OS return types, or NULL
);
/* The indexed repository as the C-family preprocessed pass's #include lookup
* sees it (#2993): `root` is the directory rel_path is relative to, and a
* quoted include resolves only to a file is_indexed() accepts -- one of the
* files the index run covers -- so the expansion is a function of the indexed
* tree alone, never of the process working directory. */
typedef struct {
const char *root;
bool (*is_indexed)(const void *userdata, const char *rel_path);
const void *userdata;
} CBMIncludeScope;
/* Additive immutable-snapshot entry point. NULL preserves legacy behavior.
* Snapshot must outlive this call; returned data is independently result-owned.
* A non-OK result status is an explicit configured-mapping failure, not a
* successful empty inventory. error_msg may be NULL on diagnostic OOM. */
* successful empty inventory. error_msg may be NULL on diagnostic OOM.
* include_scope: where quoted #include names resolve (preprocessor.h); NULL
* resolves none. */
CBMFileResult *cbm_extract_file_ex_with_tests(
const char *source, int source_len, CBMLanguage language, const char *project,
const char *rel_path, int64_t timeout_micros, const char **extra_defines,
const char **include_paths, const CBMMacroTable *macro_table,
const CBMReturnTypeTable *return_type_table, const cbm_test_declarations_t *test_declarations);
const CBMReturnTypeTable *return_type_table, const cbm_test_declarations_t *test_declarations,
const CBMIncludeScope *include_scope);
// Free all memory associated with a result.
void cbm_free_result(CBMFileResult *result);
+382 -24
View File
@@ -5,13 +5,267 @@
#include "preprocessor.h"
#include "vendored/simplecpp/simplecpp.h"
#include <deque>
#include <map>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
#include <cstring>
#include <cstdlib>
#include <cstdint>
// ── #include lookup against the indexed repository root (#2993) ────────────
// simplecpp looks a quoted header up as `dirPath(includer) + name` and opens it
// relative to the process working directory, so the expansion depended on the
// directory the indexer was started from. Here every #include and
// __has_include of the file, and of each header it reaches, is resolved before
// simplecpp runs: a quoted name in the including file's directory within the
// repository root, read through the caller's CBMIncludeReader; an angle-bracket
// name resolves to nothing, as no include directories are configured. The
// result is stored in simplecpp's FileDataCache under the exact key its own
// lookup (FileDataCache::get) computes, so simplecpp opens no file itself and
// the expansion is a function of the indexed tree alone.
namespace {
// A computed `#include MACRO` line names its header only after macro expansion,
// which no lookup made ahead of simplecpp can see. Its directive is renamed to
// one simplecpp does not know, and an unknown directive includes nothing.
const char kUnresolvedIncludeDirective[] = "cbm_unresolved_include";
// True when `key`, the normalised name an include resolves to, names a file
// within the indexed repository root: the name was written relative to the
// including file, and every component of the normalised result names an entry
// of the tree. The rule is the same on every platform, so one tree resolves the
// same way everywhere.
bool names_tree_file(const std::string &header, const std::string &key) {
if (header.empty() || header[0] == '/' || header[0] == '\\' ||
(header.size() >= 2U && header[1] == ':')) {
return false;
}
if (key.empty() || key[0] == '/' || key.find(':') != std::string::npos ||
key.find('\\') != std::string::npos || key.find('\0') != std::string::npos) {
return false;
}
std::string::size_type start = 0;
for (;;) {
const std::string::size_type end = key.find('/', start);
const std::string part =
key.substr(start, end == std::string::npos ? std::string::npos : end - start);
if (part == "..") {
return false;
}
if (end == std::string::npos) {
return true;
}
start = end + 1U;
}
}
// The first non-comment token after `tok` on the same line, or nullptr.
simplecpp::Token *next_on_line(simplecpp::Token *tok) {
simplecpp::Token *next = tok->next;
while (next && next->comment) {
next = next->next;
}
return sameline(tok, next) ? next : nullptr;
}
class IncludeTree {
public:
IncludeTree(const CBMIncludeReader *reader, std::vector<std::string> &files,
simplecpp::FileDataCache &cache)
: reader_(reader), files_(files), cache_(cache) {}
// Resolve everything the main file reaches. Every file name its own
// tokens registered (itself, and names from its #line directives) is the
// main file's; every later one belongs to a header.
void add_main_file(simplecpp::TokenList &tokens) {
first_header_file_ = static_cast<unsigned int>(files_.size());
prepare(tokens);
while (!pending_.empty()) {
Pending next = std::move(pending_.front());
pending_.pop_front();
prepare(next.tokens);
cache_.insert(simplecpp::FileData{next.key, std::move(next.tokens)});
}
}
// Tokens whose file index is at or above this came from a header.
unsigned int first_header_file() const {
return first_header_file_;
}
// The repository files the lookup read, sorted (the map's order).
void read_files(std::vector<std::string> *out) const {
for (std::map<std::string, bool>::const_iterator it = loaded_.begin(); it != loaded_.end();
++it) {
if (it->second) {
out->push_back(it->first);
}
}
}
private:
struct Pending {
std::string key;
simplecpp::TokenList tokens;
};
// Rewrite one file's __has_include tests and computed includes, and
// resolve the headers its #include lines name.
void prepare(simplecpp::TokenList &tokens) {
for (simplecpp::Token *tok = tokens.front(); tok; tok = tok->next) {
if (tok->str() == HAS_INCLUDE) {
rewrite_has_include(tokens, tok);
} else if (tok->op == '#' && !sameline(tok->previousSkipComments(), tok)) {
simplecpp::Token *directive = next_on_line(tok);
if (directive && directive->str() == INCLUDE) {
resolve_include(directive);
}
}
}
}
// `#include "name"` / `#include <name>` as simplecpp's preprocess reads it:
// the first non-comment token after the directive, quotes or brackets
// stripped.
void resolve_include(simplecpp::Token *directive) {
const simplecpp::Token *operand = next_on_line(directive);
if (!operand) {
return; // no header name: simplecpp reports it and opens nothing
}
if (operand->name) {
directive->setstr(kUnresolvedIncludeDirective);
return;
}
const std::string &spelled = operand->str();
if (spelled.size() <= 2U) {
return; // no header name: simplecpp reports it and opens nothing
}
(void)lookup(file_of(directive), spelled.substr(1U, spelled.size() - 2U),
spelled[0] == '<');
}
// `__has_include ( name )` (or without the parentheses) as simplecpp's #if
// evaluation reads it becomes the literal 1 or 0 of the same lookup. The
// operand of `defined` / `#ifdef` / `#ifndef` stays as it is. A form that
// does not close on its line keeps its operand, and its #if still fails to
// evaluate, as it did before.
void rewrite_has_include(simplecpp::TokenList &tokens, simplecpp::Token *tok) {
const simplecpp::Token *before = tok->previousSkipComments();
if (before && before->op == '(' && sameline(before, tok)) {
before = before->previousSkipComments();
}
if (before && sameline(before, tok) &&
(before->str() == DEFINED || before->str() == IFDEF || before->str() == IFNDEF)) {
return;
}
simplecpp::Token *operand = next_on_line(tok);
const bool par = operand && operand->op == '(';
if (par) {
operand = next_on_line(operand);
}
bool found = false;
simplecpp::Token *last = operand;
if (operand && operand->op == '<') {
while (last && last->op != '>') {
last = next_on_line(last);
}
} else if (operand && operand->str().size() > 2U && operand->str()[0] == '"') {
const std::string &spelled = operand->str();
found = lookup(file_of(tok), spelled.substr(1U, spelled.size() - 2U), false);
}
if (last && par) {
last = next_on_line(last);
if (last && last->op != ')') {
last = nullptr;
}
}
tok->setstr(found ? "1" : "0");
if (!last) {
return;
}
while (tok->next != last) {
tokens.deleteToken(tok->next);
}
tokens.deleteToken(last);
}
const std::string &file_of(const simplecpp::Token *tok) const {
static const std::string none;
return tok->location.fileIndex < files_.size() ? files_[tok->location.fileIndex] : none;
}
// Key the name the way simplecpp's FileDataCache::get does and make sure
// the cache holds that key, so get() finds it instead of opening a file.
// True when the name resolved to a file of the indexed tree.
bool lookup(const std::string &sourcefile, const std::string &header, bool system) {
if (simplecpp::isAbsolutePath(header)) {
return load(simplecpp::simplifyPath(header), false);
}
if (system) {
return false; // no include directories: get() opens nothing
}
const std::string key = simplecpp::simplifyPath(dirPath(sourcefile) + header);
return load(key, names_tree_file(header, key));
}
// Cache `key` once: its tokens when the tree holds it, an empty token list
// otherwise (the include then adds nothing). The main file never resolves
// as a header: its tokens are the expansion itself.
bool load(const std::string &key, bool in_tree) {
const std::map<std::string, bool>::const_iterator seen = loaded_.find(key);
if (seen != loaded_.end()) {
return seen->second;
}
std::string text;
const bool found = in_tree && !names_main_file(key) && read(key, &text);
loaded_[key] = found;
if (!found) {
cache_.insert(simplecpp::FileData{key, simplecpp::TokenList(files_)});
return false;
}
std::istringstream istr(text);
pending_.push_back(Pending{key, simplecpp::TokenList(istr, files_, key)});
return true;
}
bool names_main_file(const std::string &key) const {
for (unsigned int i = 0; i < first_header_file_ && i < files_.size(); i++) {
if (files_[i] == key) {
return true;
}
}
return false;
}
bool read(const std::string &key, std::string *out) const {
if (!reader_ || !reader_->read) {
return false;
}
size_t len = 0;
char *data = reader_->read(reader_->userdata, key.c_str(), &len);
if (!data) {
return false;
}
out->assign(data, len);
if (reader_->release) {
reader_->release(reader_->userdata, data);
}
return true;
}
const CBMIncludeReader *reader_;
std::vector<std::string> &files_;
simplecpp::FileDataCache &cache_;
std::map<std::string, bool> loaded_;
std::deque<Pending> pending_;
unsigned int first_header_file_ = 0;
};
} // namespace
extern "C" {
static bool has_preprocessor_work(const char *source, int source_len) {
@@ -407,11 +661,23 @@ static bool tokens_source_adjacent(const simplecpp::Token *prev, const simplecpp
// scopes, C-LSP class scopes) minted a second node for the same definition, the
// QN dedup (strcmp) missing it. A gap that held whitespace or a comment is still
// one space, as before.
static std::string stringify_source_spacing(const simplecpp::TokenList &tokens) {
//
// Tokens of an included header's body (file index at or above
// first_header_file) are left out: the header is indexed as its own file, and
// its calls, usages and writes are its own, never the includer's. What the
// header defines still shapes the main file's tokens, since macro expansions
// carry the location of their use. With no header body in the buffer, the main
// file's lines keep their own numbers, as they do when nothing is included.
static std::string stringify_source_spacing(const simplecpp::TokenList &tokens,
unsigned int first_header_file) {
std::ostringstream ret;
simplecpp::Location loc;
loc.line = 1;
const simplecpp::Token *prev = nullptr;
for (const simplecpp::Token *tok = tokens.cfront(); tok; tok = tok->next) {
if (tok->location.fileIndex >= first_header_file) {
continue;
}
if (tok->location.line < loc.line || tok->location.fileIndex != loc.fileIndex) {
ret << "\n#line " << tok->location.line << " \"" << tokens.file(tok->location)
<< "\"\n";
@@ -421,16 +687,113 @@ static std::string stringify_source_spacing(const simplecpp::TokenList &tokens)
ret << '\n';
loc.line++;
}
const simplecpp::Token *prev = tok->previous;
if (prev && prev->location.sameline(tok->location) && !tokens_source_adjacent(prev, tok)) {
ret << ' ';
}
ret << tok->str();
loc.adjust(tok->str());
prev = tok;
}
return ret.str();
}
// simplecpp drops the whole expansion when it stops at #error, or at a
// directive it cannot read or evaluate: the output list is cleared and an
// ERROR or SYNTAX_ERROR is reported.
static bool expansion_discarded(const simplecpp::TokenList &output,
const simplecpp::OutputList &outputs) {
if (!output.empty()) {
return false;
}
for (simplecpp::OutputList::const_iterator it = outputs.begin(); it != outputs.end(); ++it) {
if (it->type == simplecpp::Output::ERROR || it->type == simplecpp::Output::SYNTAX_ERROR) {
return true;
}
}
return false;
}
// One simplecpp run over the file. Every header its #include and
// __has_include lines reach is resolved through `reader` first (none when it
// is NULL); `read` (may be NULL) receives the repository files the lookup
// read, also when the run fails. Returns false when simplecpp discarded the
// expansion.
static bool preprocess_file(const std::string &src, const std::string &main_name,
const simplecpp::DUI &dui, const CBMIncludeReader *reader,
std::string *expanded, std::vector<std::string> *read) {
std::vector<std::string> files;
files.push_back(main_name);
std::istringstream istr(src);
simplecpp::TokenList rawtokens(istr, files, files[0]);
simplecpp::FileDataCache cache;
IncludeTree tree(reader, files, cache);
bool kept = false;
try {
tree.add_main_file(rawtokens);
simplecpp::TokenList output(files);
simplecpp::OutputList outputs;
simplecpp::preprocess(output, rawtokens, files, cache, dui, &outputs);
kept = !expansion_discarded(output, outputs);
*expanded = stringify_source_spacing(output, tree.first_header_file());
} catch (...) {
if (read) {
tree.read_files(read);
}
throw;
}
if (read) {
tree.read_files(read);
}
simplecpp::cleanup(cache);
return kept;
}
// The file's expansion with its includes followed. When what an include
// brought in stops simplecpp (an #error reached inside a header, a header
// directive it cannot evaluate, a failure while reading one), the file keeps
// its expansion without includes instead of losing it. `read` receives the
// repository files the include-following run read.
static std::string preprocess_with_fallback(const std::string &src, const std::string &main_name,
const simplecpp::DUI &dui,
const CBMIncludeReader *reader,
std::vector<std::string> *read) {
std::string expanded;
if (reader) {
try {
if (preprocess_file(src, main_name, dui, reader, &expanded, read)) {
return expanded;
}
} catch (...) {
// fall through to the expansion without includes
}
expanded.clear();
}
(void)preprocess_file(src, main_name, dui, nullptr, &expanded, nullptr);
return expanded;
}
// A malloc'd copy of `read` for CBMPreprocessedSource.included_files; false on
// allocation failure (everything copied so far is released by the caller).
static bool copy_included_files(const std::vector<std::string> &read, CBMPreprocessedSource *pp) {
if (read.empty()) {
return true;
}
pp->included_files = (char **)calloc(read.size(), sizeof(char *));
if (!pp->included_files) {
return false;
}
for (size_t i = 0; i < read.size(); i++) {
char *copy = (char *)malloc(read[i].size() + 1);
if (!copy) {
return false;
}
memcpy(copy, read[i].c_str(), read[i].size() + 1);
pp->included_files[i] = copy;
pp->included_count = (int)i + 1;
}
return true;
}
static bool build_line_map(const std::string &expanded, const std::string &main_file,
uint32_t *original_line_by_expanded_line,
uint8_t *belongs_to_main_file) {
@@ -472,7 +835,9 @@ static bool build_line_map(const std::string &expanded, const std::string &main_
CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_len,
const char *filename, const char **extra_defines,
const char **include_paths, int cpp_mode) {
const char **include_paths, int cpp_mode,
const CBMIncludeReader *reader) {
(void)include_paths; // lookups resolve through `reader` only (preprocessor.h)
// Run the second pass when there are directives to evaluate OR when the file
// carries export-macro-shaped identifiers to predefine empty (#1989) — a UE
// plugin header with only `#pragma once` + `#include` lines still needs it.
@@ -509,32 +874,21 @@ CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_le
dui.defines.push_back(std::string(export_cands[i]) + "=");
}
}
if (include_paths) {
for (int i = 0; include_paths[i]; i++)
dui.includePaths.push_back(include_paths[i]);
}
dui.std = cpp_mode ? "c++17" : "c11";
std::string src(source, source_len);
std::istringstream istr(src);
std::vector<std::string> files;
files.push_back(filename ? filename : "<input>");
simplecpp::TokenList rawtokens(istr, files, files[0]);
simplecpp::TokenList output(files);
simplecpp::FileDataCache filedata = simplecpp::load(rawtokens, files, dui);
simplecpp::preprocess(output, rawtokens, files, filedata, dui);
std::string result = stringify_source_spacing(output);
// Clean up loaded file data
simplecpp::cleanup(filedata);
const std::string src(source, source_len);
const std::string main_name(filename ? filename : "<input>");
std::vector<std::string> read;
std::string result = preprocess_with_fallback(src, main_name, dui, reader, &read);
CBMPreprocessedSource *pp = (CBMPreprocessedSource *)calloc(1, sizeof(*pp));
if (!pp) {
return NULL;
}
if (!copy_included_files(read, pp)) {
cbm_preprocessed_source_free(pp);
return NULL;
}
int line_count = count_expanded_lines(result);
pp->source = (char *)malloc(result.size() + 1);
pp->original_line_by_expanded_line =
@@ -546,7 +900,7 @@ CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_le
return NULL;
}
memcpy(pp->source, result.c_str(), result.size() + 1);
if (!build_line_map(result, files[0], pp->original_line_by_expanded_line,
if (!build_line_map(result, main_name, pp->original_line_by_expanded_line,
pp->belongs_to_main_file)) {
cbm_preprocessed_source_free(pp);
return NULL;
@@ -561,7 +915,7 @@ CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_le
char *cbm_preprocess(const char *source, int source_len, const char *filename,
const char **extra_defines, const char **include_paths, int cpp_mode) {
CBMPreprocessedSource *pp = cbm_preprocess_with_map(source, source_len, filename, extra_defines,
include_paths, cpp_mode);
include_paths, cpp_mode, NULL);
if (!pp) {
return NULL;
}
@@ -582,6 +936,10 @@ void cbm_preprocessed_source_free(CBMPreprocessedSource *pp) {
free(pp->source);
free(pp->original_line_by_expanded_line);
free(pp->belongs_to_main_file);
for (int i = 0; i < pp->included_count; i++) {
free(pp->included_files[i]);
}
free(pp->included_files);
free(pp);
}
+30 -3
View File
@@ -22,28 +22,55 @@ typedef struct {
uint32_t *original_line_by_expanded_line; // 1-based; 0 means directive/unmapped.
uint8_t *belongs_to_main_file; // 1-based; true only for the original input file.
int expanded_line_count;
/* The repository files the #include lookup read (repository-relative,
* sorted, each once), whether or not the run that followed includes kept
* its expansion: the expansion depends on their bytes either way. */
char **included_files;
int included_count;
} CBMPreprocessedSource;
/* Supplies the files of the indexed repository to the #include lookup of the
* preprocessed pass. rel_path is relative to the repository root, '/'-separated
* and normalised. read returns a buffer of *out_len bytes, handed back to
* release once consumed, or NULL when the repository holds no such file. */
typedef struct {
char *(*read)(const void *userdata, const char *rel_path, size_t *out_len);
void (*release)(const void *userdata, char *data);
const void *userdata;
} CBMIncludeReader;
// Collect conventional export-macro-shaped identifiers (ALL_CAPS ending in
// _API/_EXPORT/_IMPORT/_DLLEXPORT/_DEPRECATED) from the source, up to max_out
// unique names. Returns the number of names stored in out.
int cbm_export_macro_candidates(const char *source, int source_len,
char (*out)[CBM_EXPORT_MACRO_NAME_MAX], int max_out);
// Preprocess C/C++ source: expand macros, evaluate #ifdef, resolve #include.
// Preprocess C/C++ source: expand macros, evaluate #ifdef. #include is not
// followed (no include reader): every included header contributes nothing.
// Returns malloc-allocated expanded source, or NULL if no expansion needed/on failure.
// extra_defines: NULL-terminated array of "NAME=VALUE" strings (can be NULL).
// include_paths: NULL-terminated array of directory paths for #include resolution (can be NULL).
// include_paths: kept for API compatibility; not consulted (see cbm_preprocess_with_map).
// The returned string must be freed with cbm_preprocess_free().
char *cbm_preprocess(const char *source, int source_len, const char *filename,
const char **extra_defines, const char **include_paths, int cpp_mode);
// Preprocess and return source plus expanded-line -> original-line ownership map.
// filename is the file's path relative to the indexed repository root. Quoted
// #include and __has_include names resolve through `reader` only, in the
// including file's directory within the repository root; a NULL reader
// resolves nothing. Angle-bracket names and computed #include lines resolve to
// nothing, and include_paths is not consulted, so the expansion is a function
// of the indexed tree alone. The expanded buffer holds the main file's own
// tokens only: what an included header defines (macros, #if state) shapes it,
// but the header's body is not part of it; the header is indexed as its own
// file. When the run that follows includes ends in #error (or another stop that
// discards the expansion), the file falls back to its expansion without includes.
// Returns NULL if no expansion is needed or preprocessing fails.
// Free with cbm_preprocessed_source_free().
CBMPreprocessedSource *cbm_preprocess_with_map(const char *source, int source_len,
const char *filename, const char **extra_defines,
const char **include_paths, int cpp_mode);
const char **include_paths, int cpp_mode,
const CBMIncludeReader *reader);
// Free preprocessed source returned by cbm_preprocess.
void cbm_preprocess_free(char *expanded);
+1
View File
@@ -471,6 +471,7 @@ static void cr_walk(cr_ctx_t *c, CBMFileResult *r) {
cr_list(c, &r->constants);
cr_list(c, &r->global_vars);
cr_list(c, &r->macros);
cr_list(c, &r->included_files);
cr_str(c, &r->error_msg);
cr_str(c, &r->error_ranges);
cr_str(c, &r->module_doc);
+1 -1
View File
@@ -55,7 +55,7 @@ src/mcp/mcp.c 765
src/pipeline/artifact.c 38
src/pipeline/fqn.c 23
src/pipeline/lsp_resolve.h 4
src/pipeline/lsp_surface.c 5
src/pipeline/lsp_surface.c 4
src/pipeline/pass_calls.c 22
src/pipeline/pass_compile_commands.c 17
src/pipeline/pass_configlink.c 3
+45 -5
View File
@@ -775,6 +775,42 @@ typedef struct {
_Atomic bool failed;
} surface_job_t;
/* Row strings are released with cbm_store_free_lsp_surfaces (free), so they
* come from the C heap: one place allocates them. */
static char *surface_row_alloc(size_t n) {
return malloc(n);
}
static char *surface_row_strdup(const char *s) {
size_t n = strlen(s);
char *copy = surface_row_alloc(n + 1);
if (copy) {
memcpy(copy, s, n + 1);
}
return copy;
}
/* CBMFileResult.included_files, one file per line ("" when none). */
static char *surface_row_included_files(const CBMFileResult *fr) {
size_t len = 0;
for (const char *const *f = fr->included_files; f && *f; f++) {
len += strlen(*f) + 1;
}
char *out = surface_row_alloc(len + 1);
if (!out) {
return NULL;
}
size_t at = 0;
for (const char *const *f = fr->included_files; f && *f; f++) {
size_t n = strlen(*f);
memcpy(out + at, *f, n);
at += n;
out[at++] = '\n';
}
out[at] = '\0';
return out;
}
static void surface_row_build(int i, void *arg) {
surface_job_t *job = (surface_job_t *)arg;
if (atomic_load_explicit(&job->failed, memory_order_relaxed)) {
@@ -814,19 +850,23 @@ static void surface_row_build(int i, void *arg) {
cbm_free(CBM_MEM_CLASS_OTHER, json);
json = stored;
}
char *included = surface_row_included_files(fr);
cbm_pipeline_result_release(fr, loaded);
if (!json) {
if (!json || !included) {
free(json);
free(included);
atomic_store_explicit(&job->failed, true, memory_order_relaxed);
return;
}
cbm_lsp_surface_row_t *r = &job->rows[i];
r->defs_json = json; /* set first: it marks the row present for the compaction */
r->project = strdup(job->project);
r->rel_path = strdup(job->files[i].rel_path);
r->surface_sha = strdup(sha);
r->project = surface_row_strdup(job->project);
r->rel_path = surface_row_strdup(job->files[i].rel_path);
r->surface_sha = surface_row_strdup(sha);
r->ref_bloom = NULL;
r->ref_bloom_len = 0;
r->config_ctx = strdup("");
r->config_ctx = surface_row_strdup("");
r->included_files = included;
if (!r->project || !r->rel_path || !r->surface_sha || !r->config_ctx) {
atomic_store_explicit(&job->failed, true, memory_order_relaxed);
}
+7 -6
View File
@@ -1545,12 +1545,13 @@ static CBMFileResult *calls_get_or_extract(cbm_pipeline_ctx_t *ctx, int idx,
}
return NULL;
}
CBMFileResult *r =
cbm_pipeline_test_force_extract_null(ctx, fi->language)
? NULL
: cbm_extract_file_ex_with_tests(
src, slen, fi->language, ctx->project_name, fi->rel_path, CBM_EXTRACT_BUDGET,
NULL, NULL, ctx->macro_table, ctx->return_type_table, ctx->test_declarations);
CBMIncludeScope include_scope = cbm_pipeline_include_scope(ctx);
CBMFileResult *r = cbm_pipeline_test_force_extract_null(ctx, fi->language)
? NULL
: cbm_extract_file_ex_with_tests(
src, slen, fi->language, ctx->project_name, fi->rel_path,
CBM_EXTRACT_BUDGET, NULL, NULL, ctx->macro_table,
ctx->return_type_table, ctx->test_declarations, &include_scope);
free(src);
(void)cbm_pipeline_test_extraction_ok(ctx, fi->language, r);
if (r) {
+5 -4
View File
@@ -1098,15 +1098,16 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
/* Studio Export XML is transformed to one cacheable aggregate so later
* passes see the same calls/usages/semantic carriers as native UDL. */
CBMIncludeScope include_scope = cbm_pipeline_include_scope(ctx);
CBMFileResult *result =
cbm_pipeline_test_force_extract_null(ctx, lang) ? NULL
: lang == CBM_LANG_OBJECTSCRIPT_EXPORT
? cbm_pipeline_extract_objectscript_export(source, source_len, ctx->project_name,
rel, ctx->macro_table, NULL)
: cbm_extract_file_ex_with_tests(source, source_len, lang, ctx->project_name, rel,
CBM_EXTRACT_BUDGET, NULL,
NULL /* no extra defines or include paths */,
ctx->macro_table, NULL, ctx->test_declarations);
: cbm_extract_file_ex_with_tests(
source, source_len, lang, ctx->project_name, rel, CBM_EXTRACT_BUDGET, NULL,
NULL /* no extra defines or include paths */, ctx->macro_table, NULL,
ctx->test_declarations, &include_scope);
free(source);
if (!cbm_pipeline_test_extraction_ok(ctx, lang, result)) {
+2 -1
View File
@@ -1361,6 +1361,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
/* Export XML uses the same cache slot as every physical file, so its
* generated classes are composed before entering the common registry
* and resolution lifecycle. */
CBMIncludeScope include_scope = cbm_pipeline_include_scope(ec->pctx);
CBMFileResult *result =
cbm_pipeline_test_force_extract_null(ec->pctx, fi->language) ? NULL
: fi->language == CBM_LANG_OBJECTSCRIPT_EXPORT
@@ -1370,7 +1371,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
: cbm_extract_file_ex_with_tests(source, source_len, fi->language, ec->project_name,
fi->rel_path, CBM_EXTRACT_BUDGET, NULL, NULL,
ec->macro_table, ec->return_type_table,
ec->pctx->test_declarations);
ec->pctx->test_declarations, &include_scope);
uint64_t file_elapsed_ms = (extract_now_ns() - file_t0) / PP_USEC_PER_MS;
+2 -1
View File
@@ -702,12 +702,13 @@ static CBMFileResult *sem_get_or_extract(cbm_pipeline_ctx_t *ctx, int file_idx,
}
return NULL;
}
CBMIncludeScope include_scope = cbm_pipeline_include_scope(ctx);
CBMFileResult *r =
cbm_pipeline_test_force_extract_null(ctx, fi->language)
? NULL
: cbm_extract_file_ex_with_tests(source, source_len, fi->language, ctx->project_name,
fi->rel_path, CBM_EXTRACT_BUDGET, NULL, NULL, NULL,
NULL, ctx->test_declarations);
NULL, ctx->test_declarations, &include_scope);
cbm_free(CBM_MEM_CLASS_SEMANTIC, source);
(void)cbm_pipeline_test_extraction_ok(ctx, fi->language, r);
if (r) {
+3 -1
View File
@@ -477,11 +477,13 @@ int cbm_pipeline_pass_usages(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fil
errors++;
continue;
}
CBMIncludeScope include_scope = cbm_pipeline_include_scope(ctx);
result = cbm_pipeline_test_force_extract_null(ctx, files[i].language)
? NULL
: cbm_extract_file_ex_with_tests(
source, source_len, files[i].language, ctx->project_name, rel,
CBM_EXTRACT_BUDGET, NULL, NULL, NULL, NULL, ctx->test_declarations);
CBM_EXTRACT_BUDGET, NULL, NULL, NULL, NULL, ctx->test_declarations,
&include_scope);
free(source);
if (!cbm_pipeline_test_extraction_ok(ctx, files[i].language, result)) {
cbm_free_result(result);
+29
View File
@@ -1007,6 +1007,27 @@ bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) {
return true;
}
CBMHashTable *cbm_pipeline_indexed_files_new(const cbm_file_info_t *files, int file_count) {
CBMHashTable *set = cbm_ht_create((uint32_t)((size_t)file_count * PAIR_LEN + CBM_SZ_64));
for (int i = 0; set && i < file_count; i++) {
cbm_ht_set(set, files[i].rel_path, (void *)&files[i]);
}
return set;
}
static bool pipeline_file_is_indexed(const void *userdata, const char *rel_path) {
return userdata && cbm_ht_get((const CBMHashTable *)userdata, rel_path) != NULL;
}
CBMIncludeScope cbm_pipeline_include_scope(const cbm_pipeline_ctx_t *ctx) {
CBMIncludeScope scope = {
.root = ctx ? ctx->repo_path : NULL,
.is_indexed = pipeline_file_is_indexed,
.userdata = ctx ? ctx->indexed_files : NULL,
};
return scope;
}
void cbm_pipeline_set_lsp_surfaces(cbm_pipeline_t *p, cbm_lsp_surface_row_t *rows, int count) {
if (!p) {
cbm_store_free_lsp_surfaces(rows, count);
@@ -4375,6 +4396,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) {
struct timespec t0;
cbm_clock_gettime(CLOCK_MONOTONIC, &t0);
cbm_path_alias_collection_t *path_aliases = NULL;
CBMHashTable *indexed_files = NULL;
cbm_file_hash_t *baseline_manifest = NULL;
int baseline_count = 0;
char **requested_excluded_dirs = NULL;
@@ -4540,11 +4562,17 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) {
* when no usable configs are found — non-TS projects pay nothing. */
path_aliases =
cbm_load_path_aliases_excluded(p->repo_path, p->excluded_dirs, p->excluded_count);
indexed_files = cbm_pipeline_indexed_files_new(files, file_count);
if (!indexed_files) {
rc = CBM_NOT_FOUND;
goto cleanup;
}
/* Build shared context for pass functions */
cbm_pipeline_ctx_t ctx = {
.project_name = p->project_name,
.repo_path = p->repo_path,
.indexed_files = indexed_files,
.gbuf = p->gbuf,
.registry = p->registry,
.cancelled = p->cancelled,
@@ -4590,6 +4618,7 @@ cleanup:
cbm_registry_free(p->registry);
p->registry = NULL;
cbm_path_alias_collection_free(path_aliases);
cbm_ht_free(indexed_files); /* keys borrowed from files */
if (restore_requested_discovery) {
cbm_discover_free_excluded(p->excluded_dirs, p->excluded_count);
cbm_discover_free_ignored(p->ignored_files, p->ignored_count);
+100 -1
View File
@@ -2383,6 +2383,7 @@ static void *rehydrate_worker(void *arg) {
* correctness stays trivially reasoned. Returns heap rows via out. */
static int closure_probe_surfaces(cbm_pipeline_t *p, const char *project,
cbm_file_info_t *probe_files, int probe_count,
const CBMHashTable *indexed_files,
cbm_lsp_surface_row_t **out_rows, int *out_count) {
*out_rows = NULL;
*out_count = 0;
@@ -2398,6 +2399,7 @@ static int closure_probe_surfaces(cbm_pipeline_t *p, const char *project,
cbm_pipeline_ctx_t probe_ctx = {
.project_name = project,
.repo_path = cbm_pipeline_repo_path(p),
.indexed_files = indexed_files,
.gbuf = probe_gbuf,
.registry = NULL,
.cancelled = cbm_pipeline_cancelled_ptr(p),
@@ -3041,6 +3043,85 @@ static bool closure_metadata_current(bool metadata_current, const cbm_file_hash_
return current && old_count == new_count;
}
/* Does `list` (one repository path per line, as lsp_surface.included_files
* stores it) name a file of `delta`? Every listed file must have been tracked
* by the previous generation: a file it did not hash cannot be proven
* unchanged, which *untracked reports. */
static bool closure_list_names_delta(const char *list, const CBMHashTable *delta,
const CBMHashTable *stored_by_path, bool *untracked) {
bool hit = false;
char name[CBM_SZ_4K];
for (const char *at = list; at && *at;) {
const char *end = strchr(at, '\n');
size_t n = end ? (size_t)(end - at) : strlen(at);
if (n > 0) {
if (n >= sizeof(name)) {
*untracked = true;
return hit;
}
memcpy(name, at, n);
name[n] = '\0';
if (!cbm_ht_get(stored_by_path, name)) {
*untracked = true;
}
if (cbm_ht_get(delta, name)) {
hit = true;
}
}
at = end ? end + 1 : at + n;
}
return hit;
}
/* The includers of a changed or deleted file (#2993). A C-family file's
* preprocessed pass reads the repository files its #include lines name, so
* its graph changes with their bytes even when its own bytes do not. Every
* stored surface row lists the files its file's lookup read, transitively
* (the headers a header includes are on the includer's own list), so one pass
* over the rows finds every includer. Each joins the delta as a changed file
* and goes through the same surface comparison as the others. `paths` holds
* the changed files, then the deleted ones; the includers are inserted after
* the changed files. Linear in the total length of the lists. Returns a
* decline reason, or NULL. */
static const char *closure_add_includers(const cbm_lsp_surface_row_t *rows, int row_count,
const CBMHashTable *stored_by_path,
const CBMHashTable *files_by_path, char **paths,
int *n_changed, int n_deleted) {
int changed = *n_changed;
CBMHashTable *delta =
cbm_ht_create((uint32_t)((size_t)(changed + n_deleted) * PAIR_LEN + CBM_SZ_64));
char **found = (char **)cbm_alloc(CBM_MEM_CLASS_OTHER,
(size_t)(row_count > 0 ? row_count : 1) * sizeof(char *));
const char *why = !delta || !found ? "alloc" : NULL;
for (int i = 0; !why && i < changed + n_deleted; i++) {
cbm_ht_set(delta, paths[i], paths[i]);
}
int found_count = 0;
for (int i = 0; !why && i < row_count; i++) {
bool untracked = false;
bool hit =
closure_list_names_delta(rows[i].included_files, delta, stored_by_path, &untracked);
if (untracked) {
why = "include_untracked";
} else if (hit && !cbm_ht_get(delta, rows[i].rel_path)) {
if (!cbm_ht_get(files_by_path, rows[i].rel_path)) {
why = "includer_not_discovered";
} else {
found[found_count++] = (char *)rows[i].rel_path;
}
}
}
if (!why && found_count > 0) {
memmove(paths + changed + found_count, paths + changed, (size_t)n_deleted * sizeof(char *));
memcpy(paths + changed, found, (size_t)found_count * sizeof(char *));
*n_changed = changed + found_count;
cbm_log_info("incremental.closure_includers", "includers", itoa_buf(found_count));
}
cbm_free(CBM_MEM_CLASS_OTHER, found);
cbm_ht_free(delta); /* keys borrowed */
return why;
}
/* Decide whether this manifest delta is closure-repairable, and if so build
* the plan. Returns 1 with *plan filled, or 0 (decline → FULL) with the
* reason logged. Every early exit declines; nothing here mutates state. */
@@ -3224,6 +3305,12 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p
for (int i = 0; i < plan->stored_count; i++) {
cbm_ht_set(rows_by_path, plan->stored_rows[i].rel_path, &plan->stored_rows[i]);
}
decline = closure_add_includers(plan->stored_rows, plan->stored_count, stored_by_path,
files_by_path, changed_paths, &n_changed, n_deleted);
if (decline) {
cbm_ht_free(rows_by_path);
goto done;
}
probe_files =
(cbm_file_info_t *)calloc((size_t)(n_changed ? n_changed : 1), sizeof(cbm_file_info_t));
@@ -3242,7 +3329,7 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p
probe_files[i] = *info;
}
int probe_rc = n_changed > 0 ? closure_probe_surfaces(p, project, probe_files, n_changed,
&probe_rows, &probe_count)
files_by_path, &probe_rows, &probe_count)
: 0;
if (probe_rc != 0) {
cbm_ht_free(rows_by_path);
@@ -3614,6 +3701,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
cbm_gbuf_t *gbuf = NULL;
cbm_registry_t *registry = NULL;
cbm_path_alias_collection_t *path_aliases = NULL;
CBMHashTable *indexed_files = NULL;
CBMHashTable *stale_surface_set = NULL;
closure_resolve_t cr = {0};
bool cr_arena_live = false;
@@ -3826,6 +3914,11 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
cbm_log_error("delta.err", "phase", "doc_link_scopes");
goto out;
}
indexed_files = cbm_pipeline_indexed_files_new(files, file_count);
if (!indexed_files) {
cbm_log_error("delta.err", "phase", "indexed_files");
goto out;
}
/* The purge already removed the closure and deleted files, so every File
* row left in the stage is a file the passes will not see. A failed read
* is a delta-route failure: the maps would silently diverge from a full
@@ -3839,6 +3932,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
cbm_pipeline_ctx_t ctx = {
.project_name = project,
.repo_path = cbm_pipeline_repo_path(p),
.indexed_files = indexed_files,
.gbuf = gbuf,
.registry = registry,
.cancelled = cbm_pipeline_cancelled_ptr(p),
@@ -4181,6 +4275,7 @@ out:
cbm_registry_free(registry);
}
cbm_path_alias_collection_free(path_aliases);
cbm_ht_free(indexed_files); /* keys borrowed from files */
if (gbuf) {
cbm_gbuf_free(gbuf);
}
@@ -4594,10 +4689,12 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
cbm_path_alias_collection_t *path_aliases =
cbm_load_path_aliases_excluded(cbm_pipeline_repo_path(p), excluded_dirs, excluded_count);
CBMHashTable *indexed_files = cbm_pipeline_indexed_files_new(files, file_count);
cbm_pipeline_ctx_t ctx = {
.project_name = project,
.repo_path = cbm_pipeline_repo_path(p),
.indexed_files = indexed_files,
.gbuf = existing,
.registry = registry,
.cancelled = cbm_pipeline_cancelled_ptr(p),
@@ -4685,6 +4782,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
free(changed_files);
cbm_registry_free(registry);
cbm_path_alias_collection_free(path_aliases);
cbm_ht_free(indexed_files); /* keys borrowed from files */
incr_free_edge_capture(&edge_cap);
cbm_store_free_coverage(old_cov, old_cov_count);
free_mode_skipped(mode_skipped, mode_skipped_count);
@@ -4784,6 +4882,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
free(changed_files);
cbm_registry_free(registry);
cbm_path_alias_collection_free(path_aliases);
cbm_ht_free(indexed_files); /* keys borrowed from files */
/* Re-link inbound cross-file edges that the purge orphaned. Runs after
* re-resolution AND post-passes so the freshly re-created target nodes
+11
View File
@@ -458,6 +458,10 @@ typedef struct cbm_tf_release {
typedef struct {
const char *project_name; /* borrowed from pipeline */
const char *repo_path; /* borrowed from pipeline */
/* The rel_path of every file of the run's whole discovery (keys borrowed
* from its file list), also when a pass is handed only some of them: the
* files a C-family #include may resolve to (#2993). NULL: none resolves. */
const CBMHashTable *indexed_files;
/* An incremental run: the unchanged files whose earlier run declared
* image builds (docker build ...), read again by the shell pass, as
* those edges are recomputed whole. Borrowed; NULL in a full run. */
@@ -577,6 +581,13 @@ typedef struct {
int tf_release_cap;
} cbm_pipeline_ctx_t;
/* The set for cbm_pipeline_ctx_t.indexed_files: every rel_path of `files`
* (keys borrowed; free with cbm_ht_free). NULL on allocation failure. */
CBMHashTable *cbm_pipeline_indexed_files_new(const cbm_file_info_t *files, int file_count);
/* The #include scope of a pass's C-family extraction: the run's repository
* root and its indexed files. */
CBMIncludeScope cbm_pipeline_include_scope(const cbm_pipeline_ctx_t *ctx);
/* Origin-aware LSP returns only success/failure. Never retain partial output
* when its configured owner proof cannot be applied to current source. */
static inline void cbm_pipeline_test_owner_error(CBMFileResult *result) {
+11 -6
View File
@@ -428,6 +428,7 @@ static int init_schema(cbm_store_t *s) {
" defs_json TEXT NOT NULL,"
" ref_bloom BLOB,"
" config_ctx TEXT NOT NULL DEFAULT '',"
" included_files TEXT NOT NULL DEFAULT '',"
" PRIMARY KEY (project, rel_path)"
");"
/* Best-effort indexing-coverage signal (#963). One row per file the
@@ -6551,11 +6552,11 @@ int cbm_store_upsert_file_hash_batch(cbm_store_t *s, const cbm_file_hash_t *hash
static int upsert_lsp_surface_row(cbm_store_t *s, const cbm_lsp_surface_row_t *row) {
const char *sql =
"INSERT INTO lsp_surface (project, rel_path, surface_sha, defs_json, ref_bloom, config_ctx)"
" VALUES (?1, ?2, ?3, ?4, ?5, ?6)"
"INSERT INTO lsp_surface (project, rel_path, surface_sha, defs_json, ref_bloom, config_ctx,"
" included_files) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"
" ON CONFLICT(project, rel_path) DO UPDATE SET surface_sha = excluded.surface_sha,"
" defs_json = excluded.defs_json, ref_bloom = excluded.ref_bloom,"
" config_ctx = excluded.config_ctx";
" config_ctx = excluded.config_ctx, included_files = excluded.included_files";
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
store_set_error_sqlite(s, "lsp_surface upsert prepare");
@@ -6571,6 +6572,7 @@ static int upsert_lsp_surface_row(cbm_store_t *s, const cbm_lsp_surface_row_t *r
sqlite3_bind_null(stmt, ST_COL_5);
}
bind_text(stmt, ST_COL_6, row->config_ctx ? row->config_ctx : "");
bind_text(stmt, ST_COL_7, row->included_files ? row->included_files : "");
int rc = sqlite3_step(stmt);
sqlite3_finalize(stmt);
if (rc != SQLITE_DONE) {
@@ -6606,8 +6608,8 @@ int cbm_store_get_lsp_surfaces(cbm_store_t *s, const char *project, cbm_lsp_surf
int *count) {
*out = NULL;
*count = 0;
const char *sql = "SELECT project, rel_path, surface_sha, defs_json, ref_bloom, config_ctx"
" FROM lsp_surface WHERE project = ?1 ORDER BY rel_path";
const char *sql = "SELECT project, rel_path, surface_sha, defs_json, ref_bloom, config_ctx,"
" included_files FROM lsp_surface WHERE project = ?1 ORDER BY rel_path";
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) {
store_set_error_sqlite(s, "lsp_surface get prepare");
@@ -6644,11 +6646,13 @@ int cbm_store_get_lsp_surfaces(cbm_store_t *s, const char *project, cbm_lsp_surf
const void *bloom = sqlite3_column_blob(stmt, ST_COL_4);
int bloom_len = sqlite3_column_bytes(stmt, ST_COL_4);
const char *cfg = (const char *)sqlite3_column_text(stmt, ST_COL_5);
const char *inc = (const char *)sqlite3_column_text(stmt, ST_COL_6);
r->project = strdup(proj ? proj : "");
r->rel_path = strdup(rel ? rel : "");
r->surface_sha = strdup(sha ? sha : "");
r->defs_json = strdup(defs ? defs : "[]");
r->config_ctx = strdup(cfg ? cfg : "");
r->included_files = strdup(inc ? inc : "");
if (bloom && bloom_len > 0) {
void *copy = malloc((size_t)bloom_len);
if (copy) {
@@ -6658,7 +6662,7 @@ int cbm_store_get_lsp_surfaces(cbm_store_t *s, const char *project, cbm_lsp_surf
}
}
if (!r->project || !r->rel_path || !r->surface_sha || !r->defs_json || !r->config_ctx ||
(bloom && bloom_len > 0 && !r->ref_bloom)) {
!r->included_files || (bloom && bloom_len > 0 && !r->ref_bloom)) {
n++; /* count the partial row so the free below releases it */
oom = true;
break;
@@ -6707,6 +6711,7 @@ void cbm_store_free_lsp_surfaces(cbm_lsp_surface_row_t *rows, int count) {
free((void *)rows[i].defs_json);
free((void *)rows[i].ref_bloom);
free((void *)rows[i].config_ctx);
free((void *)rows[i].included_files);
}
free(rows);
}
+10 -2
View File
@@ -39,8 +39,11 @@ typedef struct cbm_store_read_scope cbm_store_read_scope_t;
* 6 #2360: File nodes record their declared package clause (`package`
* property); an incremental run reads it back to build the namespace
* map and the Scala top-level index for the files it does not
* re-extract. */
#define CBM_INDEX_FORMAT_VERSION 6
* re-extract.
* 7 #2993: lsp_surface rows record the repository files a C-family file's
* #include lookup read (`included_files`); an incremental run
* re-extracts the file when one of them changes. */
#define CBM_INDEX_FORMAT_VERSION 7
#define CBM_STORE_CANCELLED (-3)
#define CBM_STORE_SCAN_LIMIT (-4)
#define CBM_STORE_CALLBACK_ERR (-5)
@@ -113,6 +116,11 @@ typedef struct {
const void *ref_bloom; /* referenced-identifier bloom blob (may be NULL) */
int ref_bloom_len;
const char *config_ctx; /* governing-config context hash ("" = none) */
/* The repository files the file's C-family #include lookup read
* (CBMFileResult.included_files): sorted, one per line, "" = none. Not part
* of the surface: an incremental run re-extracts the file when one of
* them changes. */
const char *included_files;
} cbm_lsp_surface_row_t;
/* The definition variants that lie in a file (graph_buffer.c "Definition
+20 -9
View File
@@ -42,8 +42,8 @@ static const sc_query_t sc_queries[] = {
"(SELECT 1 FROM main.nodes n WHERE n.id=e.target_id));"},
{"file_hashes", "SELECT rel_path,sha256,size FROM main.file_hashes WHERE project=?1 "
"ORDER BY 1,2,3;"},
{"lsp_surface", "SELECT rel_path,surface_sha,defs_json,ref_bloom,config_ctx FROM "
"main.lsp_surface WHERE project=?1 ORDER BY 1,2,3,4,5;"},
{"lsp_surface", "SELECT rel_path,surface_sha,defs_json,ref_bloom,config_ctx,included_files "
"FROM main.lsp_surface WHERE project=?1 ORDER BY 1,2,3,4,5,6;"},
{"index_coverage", "SELECT rel_path,kind,detail FROM main.index_coverage WHERE project=?1 "
"ORDER BY 1,2,3;"},
{"index_coverage_meta",
@@ -87,6 +87,22 @@ static void sc_frame(cbm_sha256_ctx *ctx, unsigned char tag, const void *bytes,
}
}
/* Read-only legacy databases retain their existing content digest: a table
* that gained a column is read with its earlier query when the column is
* absent. NULL for tables without an earlier shape. */
static const char *sc_legacy_query(const char *name) {
if (strcmp(name, "index_coverage_meta") == 0) {
return "SELECT index_mode,recording_status,ignored_files_stored,"
"ignored_files_total,coverage_version,hash_records_complete FROM "
"main.index_coverage_meta WHERE project=?1;";
}
if (strcmp(name, "lsp_surface") == 0) {
return "SELECT rel_path,surface_sha,defs_json,ref_bloom,config_ctx FROM "
"main.lsp_surface WHERE project=?1 ORDER BY 1,2,3,4,5;";
}
return NULL;
}
static bool sc_table(sqlite3 *db, const sc_query_t *q, const char *project, cbm_sha256_ctx *ctx) {
sqlite3_stmt *st = NULL;
if (sqlite3_prepare_v2(db, q->sql, -1, &st, NULL) != SQLITE_OK) {
@@ -105,13 +121,8 @@ static bool sc_table(sqlite3 *db, const sc_query_t *q, const char *project, cbm_
sqlite3_step(schema) == SQLITE_DONE;
return sqlite3_finalize(schema) == SQLITE_OK && absent;
}
/* Read-only legacy databases retain their existing content digest. */
if (strcmp(q->name, "index_coverage_meta") != 0 ||
sqlite3_prepare_v2(db,
"SELECT index_mode,recording_status,ignored_files_stored,"
"ignored_files_total,coverage_version,hash_records_complete FROM "
"main.index_coverage_meta WHERE project=?1;",
SC_SQL_TEXT_LENGTH, &st, NULL) != SQLITE_OK) {
const char *legacy = sc_legacy_query(q->name);
if (!legacy || sqlite3_prepare_v2(db, legacy, SC_SQL_TEXT_LENGTH, &st, NULL) != SQLITE_OK) {
sqlite3_finalize(st);
return false;
}
+38 -9
View File
@@ -18,6 +18,7 @@ enum {
GD_ALLOW_BLOB = 1 << GD_BLOB,
GD_TABLE_COUNT = 10,
GD_MAX_COLUMNS = 10,
GD_LSP_SURFACE = 5,
GD_COVERAGE_META = 7,
GD_STORE_META = 8,
GD_UNRESOLVED_CANDIDATES = 9,
@@ -78,8 +79,9 @@ static const gd_column_t gd_project_summaries[] = {{"project", GD_T, 0, 1},
{"created_at", GD_T, 0, 0},
{"updated_at", GD_T, 0, 0}};
static const gd_column_t gd_lsp_surface[] = {
{"project", GD_T, 0, 1}, {"rel_path", GD_T, 0, 2}, {"surface_sha", GD_T, 0, 0},
{"defs_json", GD_T, 0, 0}, {"ref_bloom", GD_B | GD_N, 0, 0}, {"config_ctx", GD_T, 0, 0}};
{"project", GD_T, 0, 1}, {"rel_path", GD_T, 0, 2}, {"surface_sha", GD_T, 0, 0},
{"defs_json", GD_T, 0, 0}, {"ref_bloom", GD_B | GD_N, 0, 0}, {"config_ctx", GD_T, 0, 0},
{"included_files", GD_T, 0, 0}};
static const gd_column_t gd_index_coverage[] = {{"project", GD_T, 0, 1},
{"rel_path", GD_T, 0, 2},
{"kind", GD_T, 0, 3},
@@ -119,7 +121,7 @@ static const gd_table_t gd_tables[GD_TABLE_COUNT] = {
"PRAGMA main.table_xinfo('project_summaries');",
"SELECT project,summary,source_hash,created_at,updated_at FROM main.project_summaries "
"ORDER BY CAST(project AS BLOB) ASC;"},
{"lsp_surface", gd_lsp_surface, GD_COLUMNS(gd_lsp_surface),
{"lsp_surface", gd_lsp_surface, GD_COLUMNS(gd_lsp_surface) - 1,
"PRAGMA main.table_xinfo('lsp_surface');",
"SELECT project,rel_path,surface_sha,defs_json,ref_bloom,config_ctx FROM main.lsp_surface "
"ORDER BY CAST(project AS BLOB) ASC,CAST(rel_path AS BLOB) ASC;"},
@@ -150,6 +152,14 @@ static const gd_table_t gd_coverage_meta_current = {
"ignored_files_total,coverage_version,hash_records_complete,unresolved_calls_complete "
"FROM main.index_coverage_meta ORDER BY CAST(project AS BLOB) ASC;"};
/* The same for lsp_surface: index format 7 records the files each C-family
* file's #include lookup read (`included_files`). */
static const gd_table_t gd_lsp_surface_current = {
"lsp_surface", gd_lsp_surface, GD_COLUMNS(gd_lsp_surface),
"PRAGMA main.table_xinfo('lsp_surface');",
"SELECT project,rel_path,surface_sha,defs_json,ref_bloom,config_ctx,included_files "
"FROM main.lsp_surface ORDER BY CAST(project AS BLOB) ASC,CAST(rel_path AS BLOB) ASC;"};
#undef GD_B
#undef GD_N
#undef GD_T
@@ -168,11 +178,28 @@ typedef struct {
bool have_uid;
bool have_counter;
bool have_unresolved_completeness;
bool have_included_files;
} gd_context_t;
static const gd_table_t *gd_table(const gd_context_t *g, int index) {
return index == GD_COVERAGE_META && g->have_unresolved_completeness ? &gd_coverage_meta_current
: &gd_tables[index];
if (index == GD_COVERAGE_META && g->have_unresolved_completeness) {
return &gd_coverage_meta_current;
}
if (index == GD_LSP_SURFACE && g->have_included_files) {
return &gd_lsp_surface_current;
}
return &gd_tables[index];
}
/* The newest schema a table may have, when it has a legacy one too. */
static const gd_table_t *gd_current_table(const gd_table_t *table) {
if (table == &gd_tables[GD_COVERAGE_META]) {
return &gd_coverage_meta_current;
}
if (table == &gd_tables[GD_LSP_SURFACE]) {
return &gd_lsp_surface_current;
}
return NULL;
}
typedef struct {
@@ -460,9 +487,8 @@ static bool gd_column_schema(gd_context_t *g, const gd_column_t *expected) {
static bool gd_columns(gd_context_t *g, const gd_table_t *table) {
bool seen[GD_MAX_COLUMNS] = {false};
bool coverage_meta = table == &gd_tables[GD_COVERAGE_META];
int allowed_columns =
coverage_meta ? gd_coverage_meta_current.column_count : table->column_count;
const gd_table_t *current = gd_current_table(table);
int allowed_columns = current ? current->column_count : table->column_count;
if (!gd_prepare(g, table->column_query)) {
return false;
}
@@ -498,9 +524,12 @@ static bool gd_columns(gd_context_t *g, const gd_table_t *table) {
return gd_fail(g, CBM_STORE_GRAPH_DIGEST_SCHEMA);
}
}
if (coverage_meta) {
if (table == &gd_tables[GD_COVERAGE_META]) {
g->have_unresolved_completeness = seen[table->column_count];
}
if (table == &gd_tables[GD_LSP_SURFACE]) {
g->have_included_files = seen[table->column_count];
}
return gd_finalize(g);
}
+84
View File
@@ -568,6 +568,89 @@ TEST(index_format_v5_scala_packages_rebuild) {
PASS();
}
/* Format 6 recorded no included files on its lsp_surface rows (#2993). An
* incremental refresh re-extracts the includers of a changed header from that
* record, so a format-6 index refreshed in place would keep an includer's
* edges stale after an edit to its header alone. */
static const RFile k_include_files[] = {
{"lib/m.h", "#ifndef M_H\n#define M_H\n#define DO_WORK(x) real_target(x)\n#endif\n"},
{"lib/a.c",
"#define A_LOCAL 1\n#include \"m.h\"\n\nvoid caller(int v) {\n DO_WORK(v);\n}\n"},
{"lib/t.c", "void real_target(int x) { (void)x; }\nvoid other_target(int x) { (void)x; }\n"},
};
/* CALLS edges from the function `caller` to the Function `target`. */
static int include_caller_calls(cbm_store_t *s, const char *target) {
char sql[512];
snprintf(sql, sizeof(sql),
"SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON "
"t.id=e.target_id WHERE e.type='CALLS' AND s.name='caller' AND t.name='%s' AND "
"t.label='Function'",
target);
sqlite3_stmt *st = NULL;
if (sqlite3_prepare_v2(cbm_store_get_db(s), sql, -1, &st, NULL) != SQLITE_OK) {
return -1;
}
int count = sqlite3_step(st) == SQLITE_ROW ? sqlite3_column_int(st, 0) : -1;
sqlite3_finalize(st);
return count;
}
static int include_rows_recorded(cbm_store_t *s) {
sqlite3_stmt *st = NULL;
if (sqlite3_prepare_v2(cbm_store_get_db(s),
"SELECT COUNT(*) FROM lsp_surface WHERE included_files <> ''", -1, &st,
NULL) != SQLITE_OK) {
return -1;
}
int count = sqlite3_step(st) == SQLITE_ROW ? sqlite3_column_int(st, 0) : -1;
sqlite3_finalize(st);
return count;
}
TEST(index_format_v6_included_files_rebuild) {
RProj lp;
cbm_store_t *q0 = rh_index_files(&lp, k_include_files, 3);
ASSERT_NOT_NULL(q0);
ASSERT_EQ(include_caller_calls(q0, "real_target"), 1);
ASSERT_EQ(include_rows_recorded(q0), 1);
cbm_store_close(q0);
/* The old shape: no row records the files its lookup read. */
cbm_store_t *w = cbm_store_open_path(lp.dbpath);
ASSERT_NOT_NULL(w);
ASSERT_EQ(sqlite3_exec(cbm_store_get_db(w), "UPDATE lsp_surface SET included_files = ''", NULL,
NULL, NULL),
SQLITE_OK);
ASSERT_EQ(cbm_store_set_format_version(w, 6), CBM_STORE_OK);
cbm_store_close(w);
/* Only the header changes. */
char path[700];
snprintf(path, sizeof(path), "%s/lib/m.h", lp.tmpdir);
FILE *fp = fopen(path, "wb");
ASSERT_NOT_NULL(fp);
fputs("#ifndef M_H\n#define M_H\n#define DO_WORK(x) other_target(x)\n#endif\n", fp);
fclose(fp);
char *resp = index_capture(&lp);
ASSERT_NOT_NULL(resp);
free(resp);
cbm_store_t *r = cbm_store_open_path(lp.dbpath);
ASSERT_NOT_NULL(r);
ASSERT_EQ(include_caller_calls(r, "other_target"), 1);
ASSERT_EQ(include_caller_calls(r, "real_target"), 0);
if (!strstr(g_log_buf, "format_change_reindex")) {
FAIL("a format-6 index was not routed through the full-reindex path");
}
ASSERT_EQ(include_rows_recorded(r), 1);
int format = -1;
ASSERT_EQ(cbm_store_get_format_version(r, &format), CBM_STORE_OK);
ASSERT_EQ(format, CBM_INDEX_FORMAT_VERSION);
rh_cleanup(&lp, r);
PASS();
}
SUITE(index_format) {
RUN_TEST(index_format_siblings_distinct_and_searchable);
RUN_TEST(index_format_legacy_index_rebuilds_and_repairs);
@@ -575,4 +658,5 @@ SUITE(index_format) {
RUN_TEST(index_format_swift_collisions_and_stale_calls_rebuild);
RUN_TEST(index_format_v2_images_rebuild_without_invented_host);
RUN_TEST(index_format_v5_scala_packages_rebuild);
RUN_TEST(index_format_v6_included_files_rebuild);
}
+392
View File
@@ -35512,7 +35512,399 @@ TEST(pipeline_inc_c_family_fragments) {
th_rmtree(tmp);
PASS();
}
/* ── C/C++ #include lookup against the indexed repository root (#2993) ── */
#ifdef _WIN32
#include <direct.h>
#define inc_root_chdir _chdir
#define inc_root_getcwd _getcwd
#else
#define inc_root_chdir chdir
#define inc_root_getcwd getcwd
#endif
/* Index `root` into `db` with the process working directory set to `cwd`, and
* restore the working directory afterwards. */
static int inc_root_index_from(const char *root, const char *db, const char *cwd) {
char saved[CBM_SZ_4K];
if (!inc_root_getcwd(saved, sizeof(saved)) || inc_root_chdir(cwd) != 0) {
return -1;
}
int rc = -1;
cbm_pipeline_t *p = cbm_pipeline_new(root, db, CBM_MODE_FULL);
if (p) {
(void)cbm_pipeline_set_project_name(p, "incroot");
cbm_pipeline_set_persistence(p, false);
rc = cbm_pipeline_run(p);
cbm_pipeline_free(p);
}
if (inc_root_chdir(saved) != 0) {
rc = -1;
}
return rc;
}
/* One FNV-1a digest over every node and edge row of the graph, in sorted
* order: two graphs with the same digest and row count are the same graph. */
static uint64_t inc_root_graph_digest(const char *db_path, int *rows) {
static const char *const queries[] = {
"SELECT label||'|'||qualified_name||'|'||file_path||'|'||start_line||'|'||end_line||'|'||"
"properties FROM nodes ORDER BY 1",
"SELECT e.type||'|'||s.qualified_name||'|'||t.qualified_name||'|'||e.properties FROM edges "
"e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id ORDER BY 1",
};
uint64_t h = 1469598103934665603ULL;
*rows = 0;
sqlite3 *db = NULL;
if (sqlite3_open_v2(db_path, &db, SQLITE_OPEN_READONLY, NULL) == SQLITE_OK) {
for (size_t q = 0; q < sizeof(queries) / sizeof(queries[0]); q++) {
sqlite3_stmt *stmt = NULL;
if (sqlite3_prepare_v2(db, queries[q], -1, &stmt, NULL) == SQLITE_OK) {
while (sqlite3_step(stmt) == SQLITE_ROW) {
const unsigned char *row = sqlite3_column_text(stmt, 0);
for (const unsigned char *c = row ? row : (const unsigned char *)""; *c; c++) {
h = (h ^ *c) * 1099511628211ULL;
}
h = (h ^ 0xffU) * 1099511628211ULL;
(*rows)++;
}
}
sqlite3_finalize(stmt);
}
}
sqlite3_close(db);
return h;
}
/* CALLS edges from the function named `from` to the Function named `to`. */
static int inc_root_calls(const char *db, const char *from, const char *to) {
char sql[512];
snprintf(sql, sizeof(sql),
"SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON "
"t.id=e.target_id WHERE e.type='CALLS' AND s.name='%s' AND t.name='%s' AND "
"t.label='Function'",
from, to);
return k8s_inc_count(db, sql);
}
/* #2993: the graph depended on the directory the indexer was started from:
* a quoted #include was looked up relative to the process working directory.
* The lookup now resolves against the indexed repository root. The same
* fixture, indexed from an empty directory, from a directory holding a
* different lib/m.h, and from the repository root, gives one graph, in which
* the macro of the repository's own lib/m.h expands to a call that resolves
* to its in-tree target. */
TEST(pipeline_c_include_lookup_uses_repo_root) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "%s/cbm_incroot_XXXXXX", cbm_tmpdir());
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char root[512];
char empty[512];
char other[512];
snprintf(root, sizeof(root), "%s/repo", tmp);
snprintf(empty, sizeof(empty), "%s/empty", tmp);
snprintf(other, sizeof(other), "%s/other", tmp);
write_temp_file(root, "lib/a.c",
"#define A_C_LOCAL 1\n"
"#include \"m.h\"\n"
"\n"
"void caller(int v) {\n"
" DO_WORK(v);\n"
"}\n");
write_temp_file(root, "lib/m.h",
"#ifndef M_H\n#define M_H\n#define DO_WORK(x) real_target(x)\n#endif\n");
write_temp_file(root, "lib/t.c",
"void real_target(int x) { (void)x; }\n"
"void other_target(int x) { (void)x; }\n");
write_temp_file(other, "lib/m.h",
"#ifndef M_H\n#define M_H\n#define DO_WORK(x) other_target(x)\n#endif\n");
ASSERT(cbm_mkdir_p(empty, 0755));
const char *cwds[] = {empty, other, root};
enum { RUNS = sizeof(cwds) / sizeof(cwds[0]) };
int rc[RUNS];
int rows[RUNS];
int real[RUNS];
int stray[RUNS];
uint64_t digest[RUNS];
for (int i = 0; i < RUNS; i++) {
char db[512];
snprintf(db, sizeof(db), "%s/graph%d.db", tmp, i);
rc[i] = inc_root_index_from(root, db, cwds[i]);
digest[i] = inc_root_graph_digest(db, &rows[i]);
real[i] = inc_root_calls(db, "caller", "real_target");
stray[i] = inc_root_calls(db, "caller", "other_target");
}
th_rmtree(tmp);
for (int i = 0; i < RUNS; i++) {
ASSERT_EQ(rc[i], 0);
ASSERT_GT(rows[i], 0);
ASSERT_EQ(real[i], 1);
ASSERT_EQ(stray[i], 0);
ASSERT_EQ(rows[i], rows[0]);
ASSERT(digest[i] == digest[0]);
}
PASS();
}
/* #2993: simplecpp discards a file's whole expansion when it stops at #error.
* An #error reached inside an included header must not cost the file its
* expansion: the file keeps its expansion without includes, so the call its
* own macro expands to survives. Indexed from the repository root, where the
* header is found whichever way the lookup is made. */
TEST(pipeline_c_include_error_keeps_expansion_without_includes) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "%s/cbm_incerr_XXXXXX", cbm_tmpdir());
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char root[512];
snprintf(root, sizeof(root), "%s/repo", tmp);
write_temp_file(root, "cfg.h",
"#ifndef CFG_H\n"
"#define CFG_H\n"
"#ifndef CFG_CONFIGURED\n"
"#error \"configure first\"\n"
"#endif\n"
"#endif\n");
write_temp_file(root, "z.c",
"#include \"cfg.h\"\n"
"#define RUN_STEP(x) step_target(x)\n"
"\n"
"int z_entry(int v) {\n"
" RUN_STEP(v);\n"
" return v;\n"
"}\n");
write_temp_file(root, "steps.c", "void step_target(int x) { (void)x; }\n");
char db[512];
snprintf(db, sizeof(db), "%s/graph.db", tmp);
int rc = inc_root_index_from(root, db, root);
int step_calls = inc_root_calls(db, "z_entry", "step_target");
th_rmtree(tmp);
ASSERT_EQ(rc, 0);
ASSERT_EQ(step_calls, 1);
PASS();
}
/* #2993: an included header's body is the header's own -- indexed as its own
* file, with its own lines. The includer's expansion holds only the
* includer's tokens (the header's macros still shape them), so a call or a
* write inside a header's inline function is never one of the includer's
* edges, and no edge of the includer carries a line past the includer's last
* line. Indexed from the repository root, where the header is found whichever
* way the lookup is made. */
TEST(pipeline_c_header_body_stays_with_the_header) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "%s/cbm_inchdr_XXXXXX", cbm_tmpdir());
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char root[512];
snprintf(root, sizeof(root), "%s/repo", tmp);
char header[2048];
int used = snprintf(header, sizeof(header), "#ifndef OPS_H\n#define OPS_H\n");
for (int i = 0; i < 24; i++) {
used += snprintf(header + used, sizeof(header) - (size_t)used, "/* line %d */\n", i);
}
snprintf(header + used, sizeof(header) - (size_t)used,
"extern int header_counter;\n"
"static inline int ops_inline(int v) {\n"
" header_counter = v;\n"
" return header_only_callee(v);\n"
"}\n"
"#endif\n");
write_temp_file(root, "ops.h", header);
/* user.c is 6 lines long. */
write_temp_file(root, "user.c",
"#include \"ops.h\"\n"
"#define USER_FLAG 1\n"
"\n"
"int user_fn(int v) {\n"
" return main_only_callee(v);\n"
"}\n");
write_temp_file(root, "callees.c",
"int header_counter;\n"
"int header_only_callee(int v) { return v; }\n"
"int main_only_callee(int v) { return v; }\n");
char db[512];
snprintf(db, sizeof(db), "%s/graph.db", tmp);
int rc = inc_root_index_from(root, db, root);
int from_user = k8s_inc_count(
db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON "
"t.id=e.target_id WHERE e.type IN ('CALLS','USAGE','WRITES') AND "
"s.file_path='user.c' AND t.name IN ('header_only_callee','header_counter')");
int past_end = k8s_inc_count(
db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id WHERE e.type IN "
"('CALLS','USAGE') AND s.file_path='user.c' AND "
"CAST(json_extract(e.properties,'$.line') AS INTEGER) > 6");
int own_call = k8s_inc_count(
db, "SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON "
"t.id=e.target_id WHERE e.type='CALLS' AND s.file_path='ops.h' AND "
"t.name='header_only_callee'");
int user_call = inc_root_calls(db, "user_fn", "main_only_callee");
th_rmtree(tmp);
ASSERT_EQ(rc, 0);
ASSERT_EQ(from_user, 0);
ASSERT_EQ(past_end, 0);
ASSERT_GT(own_call, 0);
ASSERT_EQ(user_call, 1);
PASS();
}
/* #2993: only a name that denotes a file of the indexed tree resolves. Each
* of the three includes of lib/a.c names a header that is not part of the
* indexed tree (the third is reached through a symbolic link, which discovery
* does not index either), so none of the three macros expands, wherever the
* indexer runs; here it runs from the repository root. */
TEST(pipeline_c_include_lookup_stays_in_repo_root) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "%s/cbm_incside_XXXXXX", cbm_tmpdir());
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char root[512];
snprintf(root, sizeof(root), "%s/repo", tmp);
write_temp_file(tmp, "m_side.h", "#define SIDE_M(x) m_target(x)\n");
write_temp_file(tmp, "n_side.h", "#define SIDE_N(x) n_target(x)\n");
write_temp_file(tmp, "l_side.h", "#define SIDE_L(x) l_target(x)\n");
char source[2048];
snprintf(source, sizeof(source),
"#define A_LOCAL 1\n"
"#include \"../../m_side.h\"\n"
"#include \"%s/n_side.h\"\n"
"#include \"l_side.h\"\n"
"\n"
"void caller(int v) {\n"
" SIDE_M(v);\n"
" SIDE_N(v);\n"
" SIDE_L(v);\n"
"}\n",
tmp);
write_temp_file(root, "lib/a.c", source);
write_temp_file(root, "lib/t.c",
"void m_target(int x) { (void)x; }\n"
"void n_target(int x) { (void)x; }\n"
"void l_target(int x) { (void)x; }\n");
#ifndef _WIN32
char link_target[512];
char link_path[512];
snprintf(link_target, sizeof(link_target), "%s/l_side.h", tmp);
snprintf(link_path, sizeof(link_path), "%s/lib/l_side.h", root);
ASSERT_EQ(symlink(link_target, link_path), 0);
#endif
char db[512];
snprintf(db, sizeof(db), "%s/graph.db", tmp);
int rc = inc_root_index_from(root, db, root);
static const char *const targets[] = {"m_target", "n_target", "l_target"};
int wrong = 0;
for (size_t i = 0; i < sizeof(targets) / sizeof(targets[0]); i++) {
int n = inc_root_calls(db, "caller", targets[i]);
if (n != 0) {
printf(" caller -> %s: %d\n", targets[i], n);
wrong++;
}
}
th_rmtree(tmp);
ASSERT_EQ(rc, 0);
ASSERT_EQ(wrong, 0);
PASS();
}
/* #2993: a C file's graph depends on the in-tree headers its includes read,
* so an edit to a header alone changes its includers' graphs. lib/a.c
* includes lib/m.h directly, lib/b.c through lib/mid.h; only lib/m.h's macro
* changes. The incremental refresh (closure repair) must re-extract both
* includers and give the graph a fresh full index gives. 60 filler files take
* the fixture past MIN_FILES_FOR_PARALLEL, so the fresh full index resolves
* calls on the same parallel path as the repair: the sequential path writes
* the same CALLS properties in another key order. */
TEST(pipeline_c_header_edit_refreshes_includers_incrementally) {
char tmp[256];
snprintf(tmp, sizeof(tmp), "%s/cbm_incedit_XXXXXX", cbm_tmpdir());
ASSERT_NOT_NULL(cbm_mkdtemp(tmp));
char root[512];
snprintf(root, sizeof(root), "%s/repo", tmp);
write_temp_file(root, "lib/m.h",
"#ifndef M_H\n#define M_H\n#define DO_WORK(x) real_target(x)\n#endif\n");
write_temp_file(root, "lib/mid.h", "#ifndef MID_H\n#define MID_H\n#include \"m.h\"\n#endif\n");
write_temp_file(root, "lib/a.c",
"#define A_LOCAL 1\n"
"#include \"m.h\"\n"
"\n"
"void caller_a(int v) {\n"
" DO_WORK(v);\n"
"}\n");
write_temp_file(root, "lib/b.c",
"#define B_LOCAL 1\n"
"#include \"mid.h\"\n"
"\n"
"void caller_b(int v) {\n"
" DO_WORK(v);\n"
"}\n");
write_temp_file(root, "lib/t.c",
"void real_target(int x) { (void)x; }\n"
"void other_target(int x) { (void)x; }\n");
write_temp_file(root, "lib/u.c", "int unrelated(void) { return 1; }\n");
for (int i = 0; i < 60; i++) {
char name[64];
char body[96];
snprintf(name, sizeof(name), "fill/fill_%02d.c", i);
snprintf(body, sizeof(body), "int fill_%02d(void) { return %d; }\n", i, i);
write_temp_file(root, name, body);
}
char db[512];
char fresh_db[512];
snprintf(db, sizeof(db), "%s/graph.db", tmp);
snprintf(fresh_db, sizeof(fresh_db), "%s/fresh.db", tmp);
int rc_full = inc_root_index_from(root, db, root);
int before_a = inc_root_calls(db, "caller_a", "real_target");
write_temp_file(root, "lib/m.h",
"#ifndef M_H\n#define M_H\n#define DO_WORK(x) other_target(x)\n#endif\n");
cbm_pipeline_incremental_test_reset_faults();
int rc_incr = inc_root_index_from(root, db, root);
cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route();
int rc_fresh = inc_root_index_from(root, fresh_db, root);
int incr_rows = 0;
int fresh_rows = 0;
uint64_t incr_digest = inc_root_graph_digest(db, &incr_rows);
uint64_t fresh_digest = inc_root_graph_digest(fresh_db, &fresh_rows);
unsigned char incr_content[32] = {0};
unsigned char fresh_content[32] = {1};
cbm_store_t *incr_store = cbm_store_open_path(db);
cbm_store_t *fresh_store = cbm_store_open_path(fresh_db);
bool contents_read =
incr_store && fresh_store &&
cbm_store_graph_content_digest(incr_store, "incroot", incr_content) == CBM_STORE_OK &&
cbm_store_graph_content_digest(fresh_store, "incroot", fresh_content) == CBM_STORE_OK;
cbm_store_close(incr_store);
cbm_store_close(fresh_store);
static const char *const callers[] = {"caller_a", "caller_b"};
int wrong = 0;
for (size_t i = 0; i < sizeof(callers) / sizeof(callers[0]); i++) {
int now = inc_root_calls(db, callers[i], "other_target");
int stale = inc_root_calls(db, callers[i], "real_target");
if (now != 1 || stale != 0) {
printf(" %s: other_target %d, real_target %d\n", callers[i], now, stale);
wrong++;
}
}
th_rmtree(tmp);
ASSERT_EQ(rc_full, 0);
ASSERT_EQ(before_a, 1);
ASSERT_EQ(rc_incr, 0);
ASSERT_EQ(route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR);
ASSERT_EQ(rc_fresh, 0);
ASSERT_EQ(wrong, 0);
ASSERT_EQ(incr_rows, fresh_rows);
ASSERT(incr_digest == fresh_digest);
ASSERT(contents_read);
ASSERT(memcmp(incr_content, fresh_content, sizeof(incr_content)) == 0);
PASS();
}
SUITE(pipeline) {
RUN_TEST(pipeline_c_header_edit_refreshes_includers_incrementally);
RUN_TEST(pipeline_c_include_lookup_uses_repo_root);
RUN_TEST(pipeline_c_include_error_keeps_expansion_without_includes);
RUN_TEST(pipeline_c_header_body_stays_with_the_header);
RUN_TEST(pipeline_c_include_lookup_stays_in_repo_root);
RUN_TEST(pipeline_inc_c_family_fragments);
RUN_TEST(pipeline_python_module_callers_sequential);
RUN_TEST(pipeline_python_module_callers_parallel);
+2 -2
View File
@@ -537,12 +537,12 @@ static pc_spill_witness_t pc_collect_spill_witness(pc_fixture_t *f) {
decl = cbm_test_declarations_parse(pc_config0, strlen(pc_config0), false);
if (!decl) goto done;
cache[0] = cbm_extract_file_ex_with_tests(plain, (int)strlen(plain), CBM_LANG_C,
PC_PROJECT, "contributor.c", 0, NULL, NULL, NULL, NULL, decl);
PC_PROJECT, "contributor.c", 0, NULL, NULL, NULL, NULL, decl, NULL);
if (!cache[0] || cache[0]->test_declarations_status != CBM_TEST_EXTRACT_OK || cache[0]->defs.count < 1)
goto done;
cbm_result_compact(cache[0]);
cache[1] = cbm_extract_file_ex_with_tests(owner, (int)strlen(owner), CBM_LANG_C,
PC_PROJECT, "parked.c", 0, NULL, NULL, NULL, NULL, decl);
PC_PROJECT, "parked.c", 0, NULL, NULL, NULL, NULL, decl, NULL);
if (!pc_result_case(cache[1], "CHECK_parked", PC_PROJECT ".parked.CHECK_parked")) goto done;
cbm_result_compact(cache[1]);
ctx.test_declarations = decl;
+1 -1
View File
@@ -33,7 +33,7 @@ static cbm_test_declarations_t *tc_config(const char *records, const char *prese
static CBMFileResult *tc_extract(const char *source, CBMLanguage language, const char *path,
const cbm_test_declarations_t *config) {
return cbm_extract_file_ex_with_tests(source, (int)strlen(source), language,
"convproj", path, 0, NULL, NULL, NULL, NULL, config);
"convproj", path, 0, NULL, NULL, NULL, NULL, config, NULL);
}
static CBMFileResult *tc_legacy(const char *source, CBMLanguage language, const char *path) {