Skip to content

fix: rebuild index when mode adds capabilities #1263

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
astandrik wants to merge 1 commit into
base: main
Choose a base branch
from
+365 −69
Open

fix: rebuild index when mode adds capabilities #1263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 78 additions & 48 deletions src/pipeline/pipeline.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ struct cbm_pipeline {
char *project_name;
cbm_git_context_t git_ctx;
char *branch_qn;
cbm_index_mode_t requested_mode;
cbm_index_mode_t mode;
atomic_int cancelled_storage;
atomic_int *cancelled;
Expand Down Expand Up @@ -268,6 +269,7 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path,
p->project_name = cbm_project_name_from_path(repo_path);
(void)cbm_git_context_resolve(repo_path, &p->git_ctx);
p->branch_qn = cbm_git_context_branch_qn(p->project_name, &p->git_ctx);
p->requested_mode = mode;
p->mode = mode;
p->persistence = false;
p->committed_nodes = -1;
Expand Down Expand Up @@ -1344,7 +1346,7 @@ static int capture_existing_adr(cbm_pipeline_t *p, const char *db_path) {
* metadata write has succeeded. */
static int try_incremental_or_delete_db(cbm_pipeline_t *p, cbm_file_info_t *files, int file_count,
const cbm_file_hash_t *baseline_manifest,
int baseline_count) {
int baseline_count, bool force_full_on_mismatch) {
char *db_path = resolve_db_path(p);
if (!db_path) {
return CBM_PIPELINE_FORCE_FULL_REINDEX;
Expand All @@ -1366,7 +1368,7 @@ static int try_incremental_or_delete_db(cbm_pipeline_t *p, cbm_file_info_t *file
}
cbm_log_info("pipeline.route", "path", "incremental_manifest");
int rc = cbm_pipeline_run_incremental(p, db_path, files, file_count, baseline_manifest,
baseline_count);
baseline_count, force_full_on_mismatch);
/* Delete the existing generation ONLY when we are about to rebuild it.
* On main this was guarded by an early `return rc` for the incremental
* path; this function has no such early return, so the delete must be
Expand Down Expand Up @@ -1415,19 +1417,20 @@ static int pipeline_mode_coverage_rank(cbm_index_mode_t mode) {
* must never erase files that the cheaper discovery intentionally skips. The
* exact-manifest pipeline therefore keeps the most comprehensive successfully
* published mode and performs any changed rebuild at that coverage level. */
static void promote_mode_to_existing_coverage(cbm_pipeline_t *p) {
static bool promote_mode_to_existing_coverage(cbm_pipeline_t *p) {
if (!p || !p->project_name) {
return;
return false;
}
char *db_path = resolve_db_path(p);
if (!db_path) {
return;
return false;
}
cbm_store_t *store = cbm_store_open_path_query(db_path);
free(db_path);
if (!store) {
return;
return false;
}
bool promoted = false;
cbm_coverage_meta_t meta = {0};
if (cbm_store_coverage_meta_get(store, p->project_name, &meta) == CBM_STORE_OK &&
meta.index_mode) {
Expand All @@ -1443,31 +1446,12 @@ static void promote_mode_to_existing_coverage(cbm_pipeline_t *p) {
cbm_log_info("pipeline.mode", "requested", pipeline_mode_name(p->mode), "effective",
pipeline_mode_name(stored_mode), "reason", "preserve_existing_coverage");
p->mode = stored_mode;
promoted = true;
}
}
cbm_store_coverage_meta_clear(&meta);
cbm_store_close(store);
}

int cbm_pipeline_refresh_artifact(cbm_pipeline_t *p, const char *db_path) {
if (!p || !db_path || !p->repo_path || !p->project_name) {
return 0;
}
bool existing = cbm_artifact_exists(p->repo_path);
if (!p->persistence && !existing) {
return 0;
}
int quality = p->persistence ? CBM_ARTIFACT_BEST : CBM_ARTIFACT_FAST;
int rc = cbm_artifact_export(db_path, p->repo_path, p->project_name, quality);
if (rc != 0) {
const char *err = cbm_artifact_export_last_error();
if (p->persistence) {
cbm_log_error("pipeline.err", "phase", "artifact_export", "err", err ? err : "unknown");
return rc;
}
cbm_log_warn("artifact.refresh_failed", "err", err ? err : "unknown");
}
return 0;
return promoted;
}

/* Defined below, next to the other publication helpers. */
Expand Down Expand Up @@ -1987,8 +1971,8 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas
cbm_pipeline_free_semantic_manifest(manifest, manifest_count);
if (rc != 0) {
/* db_path is this function's strdup (resolve_db_path); every return
* must release it -- refresh_artifact below only borrows it. LSan on
* the Linux leg caught exactly this pair of exits leaking. */
* must release it. LSan on the Linux leg caught exactly this pair of
* exits leaking. */
free(db_path);
return rc;
}
Expand All @@ -2001,12 +1985,8 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_hash_t *bas
free(p->saved_adr);
p->saved_adr = NULL;

/* The SQLite generation is the commit point. Automatic refresh of an
* existing artifact is best-effort, but an explicitly requested artifact
* is caller-visible and must report an export failure. */
int artifact_rc = cbm_pipeline_refresh_artifact(p, db_path);
free(db_path);
return artifact_rc;
return 0;
}

/* Run githistory pass. */
Expand Down Expand Up @@ -2130,20 +2110,26 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx,
return rc;
}

static int cbm_pipeline_run_staged(cbm_pipeline_t *p, bool *was_incremental) {
static int cbm_pipeline_run_staged(cbm_pipeline_t *p) {
if (!p) {
return CBM_NOT_FOUND;
}
*was_incremental = false;

CBM_PROF_START(t_pipeline_total);
struct timespec t0;
cbm_clock_gettime(CLOCK_MONOTONIC, &t0);
cbm_path_alias_collection_t *path_aliases = NULL;
cbm_file_hash_t *baseline_manifest = NULL;
int baseline_count = 0;
char **requested_excluded_dirs = NULL;
int requested_excluded_count = 0;
cbm_ignored_file_t *requested_ignored_files = NULL;
int requested_ignored_count = 0;
int requested_ignored_total = 0;
bool restore_requested_discovery = false;

promote_mode_to_existing_coverage(p);
p->mode = p->requested_mode;
bool mode_promoted = promote_mode_to_existing_coverage(p);

/* cbm_pipeline_new() may precede the actual run by an arbitrary interval.
* Refresh once here, then use this exact snapshot for both Branch graph
Expand All @@ -2166,7 +2152,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p, bool *was_incremental) {
/* Phase 1: Discover files */
CBM_PROF_START(t_discover);
cbm_discover_opts_t opts = {
.mode = p->mode,
.mode = p->requested_mode,
.ignore_file = NULL,
.max_file_size = 0,
};
Expand Down Expand Up @@ -2200,26 +2186,62 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p, bool *was_incremental) {
/* Snapshot every semantic input once before routing/extraction. The same
* bytes drive exact no-op comparison and are checked against a fresh
* rediscovery immediately before any replacement is published. */
rc = cbm_pipeline_build_semantic_manifest(p->project_name, p->repo_path, files, file_count,
p->excluded_dirs, p->excluded_count, &p->git_ctx,
p->userconfig, &baseline_manifest, &baseline_count);
rc = mode_promoted
? cbm_pipeline_build_fresh_semantic_manifest(p->project_name, p->repo_path, p->mode,
&baseline_manifest, &baseline_count)
: cbm_pipeline_build_semantic_manifest(p->project_name, p->repo_path, files,
file_count, p->excluded_dirs, p->excluded_count,
&p->git_ctx, p->userconfig, &baseline_manifest,
&baseline_count);
if (rc != 0) {
rc = CBM_PIPELINE_ABORT_PRESERVE_DB;
goto cleanup;
}

/* Check for existing DB → try incremental or delete for reindex */
rc = try_incremental_or_delete_db(p, files, file_count, baseline_manifest, baseline_count);
rc = try_incremental_or_delete_db(p, files, file_count, baseline_manifest, baseline_count,
mode_promoted);
if (rc == CBM_PIPELINE_ABORT_PRESERVE_DB || rc == CBM_PIPELINE_PERSIST_FAILED) {
goto cleanup;
}
if (rc >= 0) {
*was_incremental = true;
goto cleanup;
}
if (rc != CBM_PIPELINE_FORCE_FULL_REINDEX) {
goto cleanup;
}

/* A changed downgrade rebuilds the complete graph at the stored effective
* mode. Keep the requested discovery lists to report the caller's scope. */
if (mode_promoted) {
cbm_discover_free(files, file_count);
files = NULL;
file_count = 0;

requested_excluded_dirs = p->excluded_dirs;
requested_excluded_count = p->excluded_count;
requested_ignored_files = p->ignored_files;
requested_ignored_count = p->ignored_count;
requested_ignored_total = p->ignored_total;
restore_requested_discovery = true;

p->excluded_dirs = NULL;
p->excluded_count = 0;
p->ignored_files = NULL;
p->ignored_count = 0;
p->ignored_total = 0;

opts.mode = p->mode;
rc = cbm_discover_ex2(p->repo_path, &opts, &files, &file_count, &p->excluded_dirs,
&p->excluded_count, &p->ignored_files, &p->ignored_count,
&p->ignored_total);
cbm_log_info("pipeline.rediscover", "requested_mode", pipeline_mode_name(p->requested_mode),
"effective_mode", pipeline_mode_name(p->mode), "files", itoa_buf(file_count));
if (rc != 0 || check_cancel(p)) {
rc = CBM_NOT_FOUND;
goto cleanup;
}
}
cbm_log_info("pipeline.route", "path", "full");

/* Phase 2: Create graph buffer and registry */
Expand Down Expand Up @@ -2269,6 +2291,15 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p, bool *was_incremental) {
cbm_registry_free(p->registry);
p->registry = NULL;
cbm_path_alias_collection_free(path_aliases);
if (restore_requested_discovery) {
cbm_discover_free_excluded(p->excluded_dirs, p->excluded_count);
cbm_discover_free_ignored(p->ignored_files, p->ignored_count);
p->excluded_dirs = requested_excluded_dirs;
p->excluded_count = requested_excluded_count;
p->ignored_files = requested_ignored_files;
p->ignored_count = requested_ignored_count;
p->ignored_total = requested_ignored_total;
}
/* Clear and free user extension config */
cbm_set_user_lang_config(NULL);
cbm_userconfig_free(p->userconfig);
Expand Down Expand Up @@ -2431,7 +2462,7 @@ static int seal_staging_db(const char *staging_path) {
return rc;
}

static int export_after_publish(cbm_pipeline_t *p, const char *final_path, bool was_incremental) {
static int export_after_publish(cbm_pipeline_t *p, const char *final_path) {
if (p->persistence) {
CBM_PROF_START(t_art);
int rc = cbm_artifact_export(final_path, p->repo_path, p->project_name, CBM_ARTIFACT_BEST);
Expand All @@ -2442,7 +2473,7 @@ static int export_after_publish(cbm_pipeline_t *p, const char *final_path, bool
}
return rc;
}
if (was_incremental && p->repo_path && cbm_artifact_exists(p->repo_path)) {
if (p->repo_path && cbm_artifact_exists(p->repo_path)) {
(void)cbm_artifact_export(final_path, p->repo_path, p->project_name, CBM_ARTIFACT_FAST);
}
return 0;
Expand Down Expand Up @@ -2484,8 +2515,7 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
free(final_path);
return CBM_NOT_FOUND;
}
bool was_incremental = false;
int rc = cbm_pipeline_run_staged(p, &was_incremental);
int rc = cbm_pipeline_run_staged(p);
free(p->db_path);
p->db_path = configured_db_path;

Expand Down Expand Up @@ -2555,7 +2585,7 @@ int cbm_pipeline_run(cbm_pipeline_t *p) {
return CBM_PIPELINE_PERSIST_FAILED;
}

rc = export_after_publish(p, final_path, was_incremental);
rc = export_after_publish(p, final_path);
free(staging_path);
free(final_path);
return rc;
Expand Down
27 changes: 15 additions & 12 deletions src/pipeline/pipeline_incremental.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ enum { INCR_RING_BUF = 4, INCR_RING_MASK = 3, INCR_TS_BUF = 24 };
#include "pipeline/pipeline.h"
#include <stdio.h>
#include <time.h>
#include "pipeline/artifact.h"
#include "pipeline/lsp_surface.h"
#include "pipeline/pass_lsp_cross.h"
#include "sqlite3.h"
Expand Down Expand Up @@ -1466,7 +1465,7 @@ static int run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_file
* generation boundary as full indexing. */
static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *project,
atomic_int *cancelled, const cbm_file_hash_t *manifest,
int manifest_count, const char *adr_content, const char *repo_path,
int manifest_count, const char *adr_content,
const cbm_coverage_row_t *cov, int cov_count,
const cbm_coverage_meta_t *meta_template,
const cbm_lsp_surface_row_t *surface_rows, int surface_row_count) {
Expand All @@ -1492,11 +1491,6 @@ static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *p
if (rc != 0) {
return rc;
}

/* Auto-update artifact if one already exists (persistence was enabled previously) */
if (repo_path && cbm_artifact_exists(repo_path)) {
cbm_artifact_export(db_path, repo_path, project, CBM_ARTIFACT_FAST);
}
return 0;
}

Expand Down Expand Up @@ -2362,7 +2356,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char

int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files,
int file_count, const cbm_file_hash_t *baseline_manifest,
int baseline_count) {
int baseline_count, bool force_full_on_mismatch) {
struct timespec t0;
cbm_clock_gettime(CLOCK_MONOTONIC, &t0);
closure_plan_t closure_plan = {0};
Expand Down Expand Up @@ -2423,7 +2417,16 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
incr_test_set_last_route(CBM_INCREMENTAL_ROUTE_NOOP);
#endif
cbm_log_info("incremental.noop", "reason", "semantic_manifest_equal");
return cbm_pipeline_refresh_artifact(p, db_path);
return 0;
}
if (force_full_on_mismatch) {
cbm_store_free_file_hashes(stored, stored_count);
cbm_store_close(store);
#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
incr_test_set_last_route(CBM_INCREMENTAL_ROUTE_FORCED_FULL);
#endif
cbm_log_info("incremental.force_full", "reason", "mode_downgrade_changed");
return CBM_PIPELINE_FORCE_FULL_REINDEX;
}
/* Manifest delta. Closure repair recomputes exactly the changed
* files plus the recorded consumers of any changed SURFACE; every
Expand Down Expand Up @@ -2848,9 +2851,9 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil
* re-parsed files have no codec output, and publishing a stale row
* would satisfy a future closure plan with yesterday's surface; an
* empty table just routes the next incremental to a full rebuild. */
int persist_rc = dump_and_persist(
existing, db_path, project, cbm_pipeline_cancelled_ptr(p), manifest, manifest_count,
saved_adr, cbm_pipeline_repo_path(p), cov, cov_n, &coverage_meta, NULL, 0);
int persist_rc =
dump_and_persist(existing, db_path, project, cbm_pipeline_cancelled_ptr(p), manifest,
manifest_count, saved_adr, cov, cov_n, &coverage_meta, NULL, 0);
cbm_pipeline_free_semantic_manifest(manifest, manifest_count);
free(saved_adr);
free(cov);
Expand Down
7 changes: 1 addition & 6 deletions src/pipeline/pipeline_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -657,7 +657,7 @@ int cbm_scan_project_env_urls_excluded(const char *root_path, cbm_env_binding_t
* files, merges into disk DB. Returns 0 on success. */
int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files,
int file_count, const cbm_file_hash_t *baseline_manifest,
int baseline_count);
int baseline_count, bool force_full_on_mismatch);

/* Exact semantic inputs for no-op/forced-full routing. The manifest contains
* every discovered source plus repository controls actually consumed by
Expand Down Expand Up @@ -752,11 +752,6 @@ int cbm_delta_patch(cbm_store_t *store, const char *project, cbm_gbuf_t *gbuf, i
const cbm_delta_saved_edge_t *snapshot, int snapshot_count);
/* discard helper shared with the delta executor (unlink stage + sidecars). */
void cbm_pipeline_discard_stage(const char *stage_path);
/* The SQLite generation is authoritative. An explicitly requested artifact is
* part of the caller-visible operation and its export error is returned;
* automatic refresh of an already-existing artifact remains best-effort. */
int cbm_pipeline_refresh_artifact(cbm_pipeline_t *p, const char *db_path);

/* Hand the pipeline the per-file LSP-surface rows serialized at the
* collect_all_defs seam (the only moment the result cache is alive).
* Takes ownership; dump_and_persist_hashes writes them into the staging
Expand Down
Loading
Loading