diff --git a/.agents/docs/2026-05-12-build-optimization-analysis.md b/.agents/docs/2026-05-12-build-optimization-analysis.md new file mode 100644 index 0000000..8d077fb --- /dev/null +++ b/.agents/docs/2026-05-12-build-optimization-analysis.md @@ -0,0 +1,216 @@ +# mcpp 构建优化深度分析报告 + +> 2026-05-12 — 模块化编译优化、缓存机制、增量编译分析 +> 基于 mcpp 0.0.10 代码库分析 + +## 1. 当前构建流程与耗时分解 + +### 1.1 mcpp 自身项目的构建数据 + +| 场景 | 耗时 | 分析 | +|---|---|---| +| 全量构建（冷） | ~21s | 合理 | +| 无改动重新 build | **~10s** | ❌ 前端开销 | +| touch 一个文件 | **~21s** | ❌ 全量重编译 | +| ninja 直接 no-op | 0.023s | 参考基线 | + +### 1.2 耗时分解 + +``` +mcpp build (touch 一个文件): +├── mcpp 前端 (~10s) +│ ├── toolchain resolve (xlings interface 子进程) +│ ├── manifest parse + dep fetch +│ ├── regex scanner (扫描所有 .cppm 文件) +│ ├── modgraph validate +│ ├── fingerprint compute +│ ├── ensure_built std module +│ ├── make_plan (BuildPlan 构建) +│ ├── BMI cache check/stage +│ └── build.ninja + compile_commands.json 生成 +│ +└── ninja 执行 (~11s) + ├── Phase 1: SCAN (1 个 .ddi 变化) ~1s + ├── Phase 2: COLLECT (build.ninja.dd 重生成) ~0.1s + ├── Phase 3: ALL 39 个模块重编译 ~10s ← 核心问题 + └── LINK ~0.5s +``` + +## 2. 两大核心问题 + +### 2.1 问题一：全局 dyndep 导致全量重编译 + +**根因**：所有编译边依赖同一个 `build.ninja.dd` 文件。 + +``` +cli.cppm 被 touch + ↓ +cli.cppm.ddi (scan) 重新生成 + ↓ +build.ninja.dd 依赖 ALL 39 个 .ddi → build.ninja.dd 被重新生成 + ↓ +所有 39 个 compile edge 都有 `| build.ninja.dd` 作为 implicit dep + ↓ +全部 39 个模块标记为 dirty → 全量重编译 +``` + +**理想行为**：只有 `cli.cppm` 和直接依赖 `mcpp.cli` 模块的文件需要重编译。 + +**ninja 的 dyndep 机制本身支持 per-file dyndep**，当前实现选择了最简单的全局方案。 + +### 2.2 问题二：mcpp 前端每次全量重算 + +即使没有任何文件改动，mcpp 仍然花 ~10s 做： +- 启动 xlings 子进程解析工具链 +- 扫描所有源文件的 module 声明 +- 生成 BuildPlan +- 重新写入 build.ninja + +这些步骤的结果在大多数增量编译场景下都不变。 + +## 3. 优化策略 + +### 3.1 策略一：per-file dyndep（影响最大） + +**目标**：改一个文件只重编译该文件及其下游依赖。 + +**方案**：将全局 `build.ninja.dd` 拆分为 per-file dyndep。 + +```ninja +# 当前（全局 dyndep）： +build build.ninja.dd : cxx_collect obj/cli.cppm.ddi obj/ui.cppm.ddi ... +build obj/cli.m.o | gcm.cache/mcpp.cli.gcm : cxx_module src/cli.cppm | build.ninja.dd + dyndep = build.ninja.dd + +# 优化后（per-file dyndep）： +build obj/cli.cppm.dd : cxx_dyndep obj/cli.cppm.ddi + restat = 1 +build obj/cli.m.o | gcm.cache/mcpp.cli.gcm : cxx_module src/cli.cppm | obj/cli.cppm.dd + dyndep = obj/cli.cppm.dd +``` + +**效果**：touch `ui.cppm` 时，只有 `ui.cppm.ddi` → `ui.cppm.dd` 变化，只有 `ui.m.o` 和依赖 `mcpp.ui` 的下游文件需要重编译。 + +**实现复杂度**：中等。需要修改 `ninja_backend.cppm` 的 emit 逻辑和 `dyndep.cppm` 的生成方式。 + +### 3.2 策略二：BMI restat + copy_if_different（减少级联重编译） + +**目标**：当模块接口不变时（只改了实现），阻止级联重编译。 + +**方案**（业界标准做法，CMake 采用）： +1. 编译器输出 BMI 到临时文件 +2. 比较临时文件与当前 BMI 内容 +3. 内容不同才覆盖（保持旧时间戳） +4. ninja `restat = 1` 检测到 BMI 未变，跳过下游 + +```ninja +rule cxx_module + command = $cxx $cxxflags -c $in -o $out.tmp && \ + (cmp -s $bmi_out.tmp $bmi_out && rm $bmi_out.tmp || mv $bmi_out.tmp $bmi_out) && \ + mv $out.tmp $out + restat = 1 +``` + +**效果**：修改 `ui.cppm` 的函数体但不改接口 → BMI 不变 → 依赖 `mcpp.ui` 的下游不重编译。 + +**GCC 注意**：GCC 每次都会重新生成 BMI 文件（即使内容相同时间戳也变），所以必须在构建系统层面做 copy_if_different。 + +### 3.3 策略三：mcpp 前端缓存（减少 10s 前端开销） + +**目标**：无改动时 mcpp 应在 <1s 内完成。 + +**方案**： + +1. **快速脏检查**：在调用 scanner/make_plan 之前，检查 `build.ninja` 是否比所有源文件更新。如果是，直接跳到 ninja 执行。 + +2. **增量 scanner**：缓存上一次的扫描结果（module graph），只重新扫描修改过的文件。 + +3. **工具链缓存**：toolchain resolve 结果缓存到 `.mcpp/cache/toolchain.json`，避免每次启动 xlings 子进程。 + +**效果**：无改动 → ~0.1s（直接 ninja no-op），改一个文件 → ~1s（增量 scan + ninja）。 + +### 3.4 策略四：Clang 两阶段编译（未来多工具链支持） + +**当前**：GCC 一次生成 BMI + .o，串行依赖。 + +**Clang 支持两阶段**： +``` +Phase 1: clang --precompile A.cppm -o A.pcm (生成 BMI) +Phase 2: clang -c A.pcm -o A.o (BMI → .o) +``` + +**好处**：A 的 BMI 就绪后，B 可以开始编译 BMI，同时 A 继续编译 .o。并行度更高。 + +**Clang 还支持 Reduced BMI**（`-fmodules-reduced-bmi`）：BMI 只包含接口信息，不包含实现细节，更小、更少级联。 + +## 4. 架构设计建议 + +### 4.1 构建后端抽象层 + +当前 `Backend` 接口已经有抽象，但实际只有 NinjaBackend。建议扩展： + +``` +Backend (abstract) +├── NinjaBackend (当前，GCC + Ninja) +├── NinjaClangBackend (未来，Clang + Ninja，两阶段编译) +├── MSBuildBackend (未来，MSVC) +└── DirectBackend (未来，无 ninja，mcpp 直接调度编译) +``` + +### 4.2 Scanner 抽象层 + +``` +ModuleScanner (abstract) +├── RegexScanner (当前，快速但不精确) +├── P1689Scanner (当前，GCC -fdeps-format=p1689r5) +├── ClangScanDepsScanner (未来，clang-scan-deps) +└── CachedScanner (装饰器，缓存上一次结果，增量更新) +``` + +### 4.3 BMI 管理层 + +``` +BmiManager +├── ProjectBmiCache (per-project target/ 目录) +├── GlobalBmiCache (当前 $MCPP_HOME/bmi/，跨项目共享) +├── BmiRestatHelper (copy_if_different + restat 机制) +└── BmiContentHash (未来，基于 BMI 内容哈希而非时间戳) +``` + +### 4.4 工具链抽象层 + +``` +Toolchain +├── GccToolchain (当前，GCC 16.1) +├── ClangToolchain (未来) +├── MsvcToolchain (未来) +└── ToolchainCache (缓存 resolve 结果) +``` + +## 5. 优先级建议 + +| 优先级 | 策略 | 预期收益 | 实现复杂度 | +|---|---|---|---| +| P0 | 前端快速脏检查 | 无改动 10s → <0.5s | 低 | +| P1 | per-file dyndep | 改一文件 21s → ~3s | 中 | +| P2 | BMI restat + copy_if_different | 改实现不改接口 → 0 级联 | 低 | +| P3 | 增量 scanner | scanner 耗时减少 80%+ | 中 | +| P4 | 工具链 resolve 缓存 | 减少 1-2s 启动开销 | 低 | +| P5 | Clang 两阶段编译支持 | 并行度提升，减少级联 | 高 | + +## 6. 业界参考 + +| 构建系统 | 模块编译策略 | 增量方案 | +|---|---|---| +| CMake 3.28+ | per-file scan + per-target collation dyndep | restat + copy_if_different | +| build2 | GCC module mapper 协议（编译时动态发现依赖） | 无需 scan 阶段 | +| xmake | 编译器原生 scan + jobgraph 并行 | 增量 scan | + +## 7. 总结 + +mcpp 的模块构建基础架构是正确的（三阶段 dyndep pipeline + BMI 缓存 + 指纹隔离），但在增量编译效率上有显著优化空间。最大的两个 win 是： + +1. **前端脏检查**（P0）— 即刻将无改动场景从 10s 降到 <0.5s +2. **per-file dyndep**（P1）— 将单文件修改场景从 21s 降到 ~3s + +这两个优化不影响正确性，不需要改变架构，可以增量实施。 diff --git a/src/build/backend.cppm b/src/build/backend.cppm index a343114..77e5ca9 100644 --- a/src/build/backend.cppm +++ b/src/build/backend.cppm @@ -21,6 +21,7 @@ struct BuildResult { std::chrono::milliseconds elapsed { 0 }; std::size_t cacheHits = 0; std::size_t cacheMisses = 0; + std::string ninjaProgram; // P0: cached for fast-path rebuilds }; struct BuildError { diff --git a/src/build/ninja_backend.cppm b/src/build/ninja_backend.cppm index 01ae2d8..eab23a9 100644 --- a/src/build/ninja_backend.cppm +++ b/src/build/ninja_backend.cppm @@ -187,8 +187,33 @@ std::string emit_ninja_string(const BuildPlan& plan) { append(" command = mkdir -p $$(dirname $out) && cp -f $in $out\n"); append(" description = STAGE $out\n\n"); + // P1: per-file dyndep rule. Converts one .ddi → .dd independently. + append("rule cxx_dyndep\n"); + append(" command = $mcpp dyndep --single --output $out $in\n"); + append(" description = DYNDEP $out\n"); + append(" restat = 1\n\n"); + + // P2: cxx_module preserves BMI timestamps when interface is unchanged. + // GCC always updates the .gcm timestamp even if content is identical. + // We backup the BMI before compilation, compile, then restore the old + // file if content is byte-identical. Combined with restat = 1 in the + // dyndep file, this prevents cascading rebuilds when only the module + // implementation changed (not the interface). + // + // $bmi_out is set per build edge to the BMI path (gcm.cache/.gcm). + // If $bmi_out is empty (no module provided), we just compile normally. append("rule cxx_module\n"); - append(" command = $cxx $cxxflags -c $in -o $out\n"); + append(" command = " + "if [ -n \"$bmi_out\" ] && [ -f \"$bmi_out\" ]; then " + "cp -p \"$bmi_out\" \"$bmi_out.bak\"; " + "fi && " + "$cxx $cxxflags -c $in -o $out && " + "if [ -n \"$bmi_out\" ] && [ -f \"$bmi_out.bak\" ] && " + "cmp -s \"$bmi_out\" \"$bmi_out.bak\"; then " + "mv \"$bmi_out.bak\" \"$bmi_out\"; " + "else " + "rm -f \"$bmi_out.bak\"; " + "fi\n"); append(" description = MOD $out\n"); if (dyndep) append(" restat = 1\n"); @@ -287,16 +312,21 @@ std::string emit_ninja_string(const BuildPlan& plan) { } append("\n"); - // ── Phase 2: collect into dyndep file. ────────────────────────── - std::string ddi_inputs; - for (auto& d : ddi_paths) - ddi_inputs += " " + d; - append("build build.ninja.dd : cxx_collect" + ddi_inputs + "\n\n"); + // ── Phase 2: per-file dyndep (P1 optimization). ──────────────── + // Each .ddi → .dd independently, so modifying one source file only + // invalidates that file's .dd and its compile edge, not all edges. + // Map ddi path → dd path for Phase 3 reference. + std::map ddi_to_dd; + for (auto& ddi : ddi_paths) { + auto dd = ddi + ".dd"; // e.g. obj/cli.cppm.ddi.dd + ddi_to_dd[ddi] = dd; + append(std::format("build {} : cxx_dyndep {}\n", dd, ddi)); + } + append("\n"); - // ── Phase 3: compile edges with dyndep. ───────────────────────── - // BMI implicit outputs are still declared statically (we know - // them from the plan); the dyndep file adds implicit BMI INPUTS - // (the requires) so ninja schedules in the right order. + // ── Phase 3: compile edges with per-file dyndep. ──────────────── + // Each compile edge references its OWN .dd file instead of a global one. + // P2: module compile edges get a $bmi_out variable for BMI preservation. for (auto& cu : plan.compileUnits) { std::string rule = pick_rule(cu.source); @@ -306,10 +336,19 @@ std::string emit_ninja_string(const BuildPlan& plan) { } out_line += std::format(" : {} {}", rule, escape_ninja_path(cu.source)); if (rule != "c_object") { - // build.ninja.dd is the dyndep file; ninja requires it as an - // implicit input (so it's built before the compile runs). - out_line += " | build.ninja.dd"; - out_line += "\n dyndep = build.ninja.dd\n"; + auto ddi = (cu.object.parent_path() / cu.source.filename()).string() + ".ddi"; + auto it = ddi_to_dd.find(ddi); + if (it != ddi_to_dd.end()) { + out_line += " | " + it->second; + out_line += "\n dyndep = " + it->second; + // P2: set bmi_out for the copy_if_different logic in cxx_module. + if (cu.providesModule) { + out_line += "\n bmi_out = " + bmi_path(*cu.providesModule); + } + out_line += "\n"; + } else { + out_line += "\n"; + } } else { out_line += "\n"; } @@ -446,6 +485,10 @@ std::expected NinjaBackend::build(const BuildPlan& plan std::string ninjaProgram = !ninjaBin.empty() ? std::format("'{}'", ninjaBin.string()) : std::string{"ninja"}; + // Record ninja binary for P0 fast-path cache. + BuildResult r; + r.ninjaProgram = ninjaProgram; + std::string cmd = std::format("{} -C '{}'", ninjaProgram, plan.outputDir.string()); if (opts.verbose) cmd += " -v"; @@ -459,7 +502,6 @@ std::expected NinjaBackend::build(const BuildPlan& plan std::fputs(out.c_str(), stdout); } - BuildResult r; r.exitCode = ok ? 0 : 1; r.elapsed = std::chrono::duration_cast( std::chrono::steady_clock::now() - t0); diff --git a/src/cli.cppm b/src/cli.cppm index a6f1f8a..c002730 100644 --- a/src/cli.cppm +++ b/src/cli.cppm @@ -1888,6 +1888,23 @@ prepare_build(bool print_fingerprint, return ctx; } +// ─── P0: build cache for fast-path rebuilds ───────────────────────── + +constexpr std::string_view kBuildCacheFile = "target/.build_cache"; + +void write_build_cache(const std::filesystem::path& projectRoot, + const std::filesystem::path& outputDir, + const std::string& ninjaProgram) { + auto path = projectRoot / kBuildCacheFile; + std::error_code ec; + std::filesystem::create_directories(path.parent_path(), ec); + std::ofstream f(path, std::ios::trunc); + if (f) { + f << outputDir.string() << '\n'; + f << ninjaProgram << '\n'; + } +} + // Compile a prepared BuildContext. Shared between `mcpp build` and `mcpp run` // so the latter doesn't call prepare_build twice (and re-print the toolchain // resolution banner). @@ -1942,10 +1959,93 @@ int run_build_plan(BuildContext& ctx, bool verbose, bool no_cache) { } } + // P0: save build cache for fast-path on next invocation. + if (!no_cache && !r->ninjaProgram.empty()) { + write_build_cache(ctx.projectRoot, ctx.outputDir, r->ninjaProgram); + } + mcpp::ui::finished("release", r->elapsed); return 0; } +// ─── P0 fast-path: skip prepare_build when build.ninja is fresh ────── +// +// On a successful build, we write `target/.build_cache` containing the +// outputDir path. On the next invocation, if build.ninja in that dir +// is newer than all source files and mcpp.toml, we invoke ninja directly +// without re-running the scanner, make_plan, or emit phases. +// +// This reduces no-change builds from ~10s to <0.5s. + +// Try to fast-path: if build.ninja is newer than all inputs, just run ninja. +// Returns exit code on fast-path, or nullopt if full rebuild needed. +std::optional try_fast_build(const std::filesystem::path& projectRoot, + bool verbose, bool no_cache) { + if (no_cache) return std::nullopt; + + auto cachePath = projectRoot / kBuildCacheFile; + std::error_code ec; + if (!std::filesystem::exists(cachePath, ec)) return std::nullopt; + + std::ifstream f(cachePath); + std::string outputDirStr, ninjaProgram; + if (!std::getline(f, outputDirStr) || outputDirStr.empty()) return std::nullopt; + if (!std::getline(f, ninjaProgram) || ninjaProgram.empty()) return std::nullopt; + std::filesystem::path outputDir(outputDirStr); + + auto ninjaPath = outputDir / "build.ninja"; + if (!std::filesystem::exists(ninjaPath, ec)) return std::nullopt; + + auto ninjaTime = std::filesystem::last_write_time(ninjaPath, ec); + if (ec) return std::nullopt; + + // Check mcpp.toml + auto tomlPath = projectRoot / "mcpp.toml"; + auto tomlTime = std::filesystem::last_write_time(tomlPath, ec); + if (ec || tomlTime > ninjaTime) return std::nullopt; + + // Check all source files under src/ + auto srcDir = projectRoot / "src"; + if (std::filesystem::exists(srcDir, ec)) { + for (auto& entry : std::filesystem::recursive_directory_iterator(srcDir, ec)) { + if (!entry.is_regular_file()) continue; + auto ext = entry.path().extension().string(); + if (ext != ".cppm" && ext != ".cpp" && ext != ".cc" && + ext != ".cxx" && ext != ".c" && ext != ".h" && ext != ".hpp") + continue; + auto ft = std::filesystem::last_write_time(entry.path(), ec); + if (ec || ft > ninjaTime) return std::nullopt; + } + } + + // All inputs are older than build.ninja → fast-path: just run ninja. + std::string cmd = std::format("{} -C '{}'", ninjaProgram, outputDir.string()); + if (verbose) cmd += " -v"; + cmd += " 2>&1"; + + auto t0 = std::chrono::steady_clock::now(); + std::string out; + FILE* pipe = popen(cmd.c_str(), "r"); + if (!pipe) return std::nullopt; + char buf[4096]; + while (std::fgets(buf, sizeof(buf), pipe)) { + out += buf; + if (verbose) std::fputs(buf, stdout); + } + int status = pclose(pipe); + bool ok = (status == 0); + if (!ok) { + if (!verbose) std::fputs(out.c_str(), stdout); + // Ninja failed — fall back to full rebuild (stale build.ninja?) + return std::nullopt; + } + + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0); + mcpp::ui::finished("release", elapsed); + return 0; +} + int cmd_build(const mcpplibs::cmdline::ParsedArgs& parsed) { bool verbose = parsed.is_flag_set("verbose"); bool print_fp = parsed.is_flag_set("print-fingerprint"); @@ -1955,6 +2055,16 @@ int cmd_build(const mcpplibs::cmdline::ParsedArgs& parsed) { if (auto t = parsed.value("target")) ov.target_triple = *t; ov.force_static = parsed.is_flag_set("static"); + // P0: try fast-path if inputs haven't changed. + if (!print_fp && ov.target_triple.empty() && !ov.force_static) { + auto root = find_manifest_root(std::filesystem::current_path()); + if (root) { + if (auto rc = try_fast_build(*root, verbose, no_cache)) { + return *rc; + } + } + } + auto ctx = prepare_build(print_fp, /*includeDevDeps=*/false, /*extraTargets=*/{}, ov); if (!ctx) { std::println(stderr, "error: {}", ctx.error()); return 2; } @@ -3255,18 +3365,36 @@ int cmd_explain_action(const mcpplibs::cmdline::ParsedArgs& parsed) { } // Hidden subcommand: aggregate P1689 .ddi files into a Ninja dyndep file. -// Invoked by ninja during build (cxx_collect rule) under M3.3.b dyndep mode. +// Invoked by ninja during build (cxx_collect / cxx_dyndep rules). +// +// Multi-file mode (legacy cxx_collect): // mcpp dyndep --output ... +// +// Single-file mode (P1 per-file dyndep, cxx_dyndep rule): +// mcpp dyndep --single --output int cmd_dyndep(const mcpplibs::cmdline::ParsedArgs& parsed) { std::filesystem::path outPath = parsed.option_or_empty("output").value(); if (outPath.empty()) { std::println(stderr, "error: --output required"); return 2; } - std::vector ddis; - for (std::size_t i = 0; i < parsed.positional_count(); ++i) - ddis.emplace_back(parsed.positional(i)); - auto body = mcpp::dyndep::emit_dyndep_from_files(ddis, /*stdImports=*/{}); + + bool single = parsed.is_flag_set("single"); + + std::expected body; + if (single) { + if (parsed.positional_count() != 1) { + std::println(stderr, "error: --single requires exactly one .ddi input"); + return 2; + } + body = mcpp::dyndep::emit_dyndep_single(parsed.positional(0)); + } else { + std::vector ddis; + for (std::size_t i = 0; i < parsed.positional_count(); ++i) + ddis.emplace_back(parsed.positional(i)); + body = mcpp::dyndep::emit_dyndep_from_files(ddis, /*stdImports=*/{}); + } + if (!body) { std::println(stderr, "error: {}", body.error()); return 1; @@ -3574,6 +3702,7 @@ int run(int argc, char** argv) { .description("(internal: invoked by ninja) Emit ninja dyndep file from .ddi inputs") .option(cl::Option("output").short_name('o').takes_value().value_name("PATH") .help("Path to write dyndep file")) + .option(cl::Option("single").help("Single-file mode: one .ddi → one .dd")) .action(wrap_rc(cmd_dyndep))) ; diff --git a/src/dyndep.cppm b/src/dyndep.cppm index c598d51..ecb2d84 100644 --- a/src/dyndep.cppm +++ b/src/dyndep.cppm @@ -52,6 +52,11 @@ std::expected emit_dyndep_from_files(const std::vector& ddiPaths, const std::set& stdImports); +// P1: emit a single-unit dyndep file from one .ddi file. +// Used by the per-file dyndep mode to convert each .ddi → .dd independently. +std::expected +emit_dyndep_single(const std::filesystem::path& ddiPath); + } // namespace mcpp::dyndep namespace mcpp::dyndep { @@ -281,4 +286,30 @@ emit_dyndep_from_files(const std::vector& ddiPaths, return emit_dyndep(units, stdImports); } +std::expected +emit_dyndep_single(const std::filesystem::path& ddiPath) +{ + std::ifstream is(ddiPath); + if (!is) return std::unexpected(std::format("cannot read '{}'", ddiPath.string())); + std::string body((std::istreambuf_iterator(is)), {}); + auto u = parse_ddi(body); + if (!u) return std::unexpected(std::format("{}: {}", ddiPath.string(), u.error())); + + std::string out = "ninja_dyndep_version = 1\n"; + if (!u->primaryOutput.empty()) { + std::string line = "build " + u->primaryOutput.string() + ": dyndep"; + bool firstImplicit = true; + for (auto& r : u->requires_) { + bool selfProvides = false; + for (auto& p : u->provides) if (p == r) { selfProvides = true; break; } + if (selfProvides) continue; + if (firstImplicit) { line += " |"; firstImplicit = false; } + line += " gcm.cache/" + bmi_basename(r); + } + line += "\n restat = 1\n"; + out += line; + } + return out; +} + } // namespace mcpp::dyndep