mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-30 07:33:59 +08:00
build(nix): add per-crate crane workspace builds
Add crane-based package outputs for the main OpenShell crates and a default symlinkJoin package. The new workspace helper derives each crate's transitive workspace dependency closure, builds from minimal source trees, and declares the assets each crate needs at compile time. Build each crate in three layers: 1. crates.io dependencies with crane buildDepsOnly 2. first-party workspace dependency libraries 3. the final real crate The workspace-libs layer builds the selected package with the same `-p <crate>` selection as final so Cargo feature unification matches, but overlays a crane-generated dummy source for the leaf crate. After that layer builds, remove the dummy leaf artifacts with `cargo clean --release -p <crate>` so the final layer cannot reuse or package stub outputs. This lets leaf edits reuse cached first-party libs while still compiling and linking the real leaf crate. Add explicit `[lib]` target names and `path = "src/lib.rs"` entries to workspace crates. The Nix source minimizer keeps every member Cargo.toml but omits source trees outside the selected crate closure; explicit target paths let Cargo resolve those member manifests without relying on auto-discovery of files that are intentionally absent. They also give crane's dummy source generation a stable target shape. Guard the openshell-core build script's `.git` rerun paths so Cargo does not mark core dirty in Nix source trees where `.git` is absent. Without this, core recompiled in the final layer and cascaded into its dependents. Known limitation: the VM driver package is wired into the flake, but the Nix build does not yet provide the compressed VM runtime artifacts that openshell-driver-vm embeds. For now that crate builds via its stub-resource fallback rather than producing a fully usable VM driver package. Ignore Nix `result*` symlinks created by local builds.
This commit is contained in:
@@ -225,3 +225,6 @@ rfc.md
|
||||
|
||||
# Markdown/mermaid lint tooling deps
|
||||
scripts/lint-mermaid/node_modules/
|
||||
|
||||
# Nix
|
||||
result*
|
||||
|
||||
@@ -9,6 +9,10 @@ license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_bootstrap"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
openshell-core = { path = "../openshell-core", default-features = false }
|
||||
bollard = "0.20"
|
||||
|
||||
@@ -10,6 +10,10 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_core"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
prost = { workspace = true }
|
||||
prost-types = { workspace = true }
|
||||
|
||||
@@ -12,8 +12,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// builds where .git is absent, this silently does nothing and the binary
|
||||
// falls back to CARGO_PKG_VERSION (which is already sed-patched by the
|
||||
// build pipeline).
|
||||
println!("cargo:rerun-if-changed=../../.git/HEAD");
|
||||
println!("cargo:rerun-if-changed=../../.git/refs/tags");
|
||||
if Path::new("../../.git/HEAD").exists() {
|
||||
println!("cargo:rerun-if-changed=../../.git/HEAD");
|
||||
}
|
||||
if Path::new("../../.git/refs/tags").exists() {
|
||||
println!("cargo:rerun-if-changed=../../.git/refs/tags");
|
||||
}
|
||||
|
||||
if let Some(version) = git_version() {
|
||||
println!("cargo:rustc-env=OPENSHELL_GIT_VERSION={version}");
|
||||
|
||||
@@ -10,6 +10,10 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_driver_docker"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
openshell-core = { path = "../openshell-core", default-features = false }
|
||||
|
||||
|
||||
@@ -10,6 +10,10 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_driver_kubernetes"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "openshell-driver-kubernetes"
|
||||
path = "src/main.rs"
|
||||
|
||||
@@ -10,6 +10,10 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_driver_podman"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "openshell-driver-podman"
|
||||
path = "src/main.rs"
|
||||
|
||||
@@ -10,6 +10,10 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_ocsf"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
serde = { workspace = true }
|
||||
|
||||
@@ -10,6 +10,10 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_policy"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
openshell-core = { path = "../openshell-core", default-features = false }
|
||||
serde = { workspace = true }
|
||||
|
||||
@@ -10,6 +10,10 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_prover"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[features]
|
||||
bundled-z3 = ["z3/bundled"]
|
||||
|
||||
|
||||
@@ -10,6 +10,10 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_providers"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
glob = { workspace = true }
|
||||
openshell-core = { path = "../openshell-core", default-features = false }
|
||||
|
||||
@@ -10,6 +10,10 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_router"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
openshell-core = { path = "../openshell-core", default-features = false }
|
||||
bytes = { workspace = true }
|
||||
|
||||
@@ -10,6 +10,8 @@ license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_server_macros"
|
||||
path = "src/lib.rs"
|
||||
proc-macro = true
|
||||
|
||||
[dependencies]
|
||||
|
||||
@@ -10,6 +10,10 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_tui"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
openshell-core = { path = "../openshell-core", default-features = false }
|
||||
openshell-bootstrap = { path = "../openshell-bootstrap" }
|
||||
|
||||
@@ -10,6 +10,10 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_vfio"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
Generated
+16
@@ -1,5 +1,20 @@
|
||||
{
|
||||
"nodes": {
|
||||
"crane": {
|
||||
"locked": {
|
||||
"lastModified": 1780099841,
|
||||
"narHash": "sha256-EVZd2RsbpreRUDSi9rBwPY+ZxoyMaiEBbZxxhljbaS4=",
|
||||
"owner": "ipetkov",
|
||||
"repo": "crane",
|
||||
"rev": "0532eb17955225173906d671fb36306bdeb1e2dc",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "ipetkov",
|
||||
"repo": "crane",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
@@ -36,6 +51,7 @@
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"crane": "crane",
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs",
|
||||
"rust-overlay": "rust-overlay",
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
url = "github:oxalica/rust-overlay";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
crane.url = "github:ipetkov/crane";
|
||||
treefmt-nix = {
|
||||
url = "github:numtide/treefmt-nix";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
@@ -22,6 +23,7 @@
|
||||
flake-utils,
|
||||
nixpkgs,
|
||||
rust-overlay,
|
||||
crane,
|
||||
treefmt-nix,
|
||||
...
|
||||
}:
|
||||
@@ -32,13 +34,92 @@
|
||||
inherit system;
|
||||
overlays = [ (import rust-overlay) ];
|
||||
};
|
||||
lib = pkgs.lib;
|
||||
rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml;
|
||||
|
||||
craneLib = (crane.mkLib pkgs).overrideToolchain (_: rustToolchain);
|
||||
|
||||
# Crate-by-crate crane helpers (workspace graph, minimal per-crate
|
||||
# source, buildWorkspaceCrate). See nix/workspace.nix.
|
||||
workspace = import ./nix/workspace.nix {
|
||||
inherit lib pkgs craneLib;
|
||||
root = ./.;
|
||||
};
|
||||
inherit (workspace) buildWorkspaceCrate;
|
||||
|
||||
# z3 (found via pkg-config) and libclang (for z3-sys bindgen) are only
|
||||
# needed by crates whose closure contains openshell-prover.
|
||||
withZ3 = {
|
||||
nativeBuildInputs = [ pkgs.pkg-config ];
|
||||
buildInputs = [ pkgs.z3 ];
|
||||
env.LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib";
|
||||
};
|
||||
|
||||
# Each crate declares the compile-time assets its build needs — its own
|
||||
# plus those of its workspace deps (proto/ arrives via openshell-core,
|
||||
# providers/ via openshell-providers, registry/ via openshell-prover).
|
||||
crates = {
|
||||
openshell-cli = buildWorkspaceCrate (
|
||||
{
|
||||
dir = "openshell-cli";
|
||||
assets = [
|
||||
./proto
|
||||
./providers
|
||||
./crates/openshell-prover/registry
|
||||
];
|
||||
}
|
||||
// withZ3
|
||||
);
|
||||
openshell-server = buildWorkspaceCrate (
|
||||
{
|
||||
dir = "openshell-server";
|
||||
assets = [
|
||||
./proto
|
||||
./providers
|
||||
./crates/openshell-prover/registry
|
||||
./crates/openshell-server/migrations
|
||||
];
|
||||
}
|
||||
// withZ3
|
||||
);
|
||||
openshell-sandbox = buildWorkspaceCrate {
|
||||
dir = "openshell-sandbox";
|
||||
assets = [
|
||||
./proto
|
||||
./crates/openshell-sandbox/data
|
||||
./crates/openshell-sandbox/src/skills
|
||||
];
|
||||
};
|
||||
openshell-driver-vm = buildWorkspaceCrate {
|
||||
dir = "openshell-driver-vm";
|
||||
assets = [
|
||||
./proto
|
||||
./crates/openshell-driver-vm/scripts
|
||||
];
|
||||
};
|
||||
openshell-driver-kubernetes = buildWorkspaceCrate {
|
||||
dir = "openshell-driver-kubernetes";
|
||||
assets = [ ./proto ];
|
||||
};
|
||||
openshell-driver-podman = buildWorkspaceCrate {
|
||||
dir = "openshell-driver-podman";
|
||||
assets = [ ./proto ];
|
||||
};
|
||||
};
|
||||
|
||||
treefmtEval = treefmt-nix.lib.evalModule pkgs {
|
||||
projectRootFile = "flake.nix";
|
||||
programs.nixfmt.enable = true;
|
||||
};
|
||||
in
|
||||
{
|
||||
packages = crates // {
|
||||
default = pkgs.symlinkJoin {
|
||||
name = "openshell-0.0.0";
|
||||
paths = lib.attrValues crates;
|
||||
};
|
||||
};
|
||||
|
||||
devShells.default = pkgs.mkShell {
|
||||
packages = with pkgs; [
|
||||
rustToolchain
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Crate-by-crate crane helpers for a Cargo workspace.
|
||||
#
|
||||
# Each crate is built from a minimal source, its own code plus that of its
|
||||
# transitive workspace dependencies, and gets its own dependency cache. Crates
|
||||
# outside that closure are reduced to their Cargo.toml so cargo can resolve
|
||||
# the workspace without their source and without any Cargo.toml edits.
|
||||
# Editing one crate never rebuilds an unrelated crate, and (because crane
|
||||
# launders the source before building deps) never rebuilds any crate's
|
||||
# dependency cache.
|
||||
{
|
||||
lib,
|
||||
pkgs,
|
||||
craneLib,
|
||||
# Workspace root: holds the virtual Cargo.toml, Cargo.lock and .cargo/.
|
||||
root,
|
||||
# Member directory, relative to root.
|
||||
crateDir ? "crates",
|
||||
# Version stamped onto every crate derivation.
|
||||
version ? "0.0.0",
|
||||
}:
|
||||
let
|
||||
cratesRoot = root + "/${crateDir}";
|
||||
|
||||
# Workspace dependency graph, derived from the Cargo.tomls
|
||||
crateDirs = lib.attrNames (lib.filterAttrs (_: t: t == "directory") (builtins.readDir cratesRoot));
|
||||
|
||||
# Direct intra-workspace path-dependencies of a crate, as dir names.
|
||||
directDeps =
|
||||
dir:
|
||||
let
|
||||
manifest = builtins.fromTOML (builtins.readFile (cratesRoot + "/${dir}/Cargo.toml"));
|
||||
in
|
||||
lib.pipe (manifest.dependencies or { }) [
|
||||
lib.attrValues
|
||||
(lib.filter (v: builtins.isAttrs v && v ? path))
|
||||
(map (v: baseNameOf v.path))
|
||||
(lib.filter (d: builtins.elem d crateDirs))
|
||||
];
|
||||
|
||||
# Transitive closure of a crate within the workspace: its own dir plus every workspace dep.
|
||||
closureOf =
|
||||
dir:
|
||||
map (e: e.key) (
|
||||
builtins.genericClosure {
|
||||
startSet = [ { key = dir; } ];
|
||||
operator = e: map (key: { inherit key; }) (directDeps e.key);
|
||||
}
|
||||
);
|
||||
|
||||
# Every member's Cargo.toml, cargo must see all of them to resolve the
|
||||
# workspace even for crates whose source we leave out.
|
||||
allManifests = map (d: cratesRoot + "/${d}/Cargo.toml") crateDirs;
|
||||
|
||||
# Source tree carrying the real sources of the given crate dirs, plus every
|
||||
# member's Cargo.toml and the given assets.
|
||||
mkSrc =
|
||||
{
|
||||
dirs,
|
||||
assets ? [ ],
|
||||
}:
|
||||
lib.fileset.toSource {
|
||||
inherit root;
|
||||
fileset = lib.fileset.unions (
|
||||
[
|
||||
(root + "/Cargo.toml")
|
||||
(root + "/Cargo.lock")
|
||||
(root + "/.cargo")
|
||||
]
|
||||
++ allManifests
|
||||
++ map (d: craneLib.fileset.commonCargoSources (cratesRoot + "/${d}")) dirs
|
||||
++ assets
|
||||
);
|
||||
};
|
||||
|
||||
# Build one workspace crate (pname == dir) in three cached layers. Every layer
|
||||
# uses the SAME `-p <dir>` selection, so cargo's feature unification is
|
||||
# identical across them and the compiled artifacts are reusable:
|
||||
# 1. crates.io deps — buildDepsOnly; immune to first-party code.
|
||||
# 2. workspace-dep libs — build `-p <dir>` with the crate's OWN source
|
||||
# stubbed (real path-deps), so its libs compile with
|
||||
# the crate's real feature set and get cached.
|
||||
# 3. the crate itself — reuses 1 + 2; only the crate's own code recompiles.
|
||||
buildWorkspaceCrate =
|
||||
{
|
||||
dir,
|
||||
assets ? [ ],
|
||||
nativeBuildInputs ? [ ],
|
||||
buildInputs ? [ ],
|
||||
env ? { },
|
||||
}:
|
||||
let
|
||||
closure = closureOf dir;
|
||||
workspaceDeps = lib.filter (d: d != dir) closure;
|
||||
common = {
|
||||
pname = dir;
|
||||
inherit
|
||||
version
|
||||
nativeBuildInputs
|
||||
buildInputs
|
||||
env
|
||||
;
|
||||
strictDeps = true;
|
||||
# Build only, skip the cargo test/check phase for now.
|
||||
doCheck = false;
|
||||
cargoExtraArgs = "--locked -p ${dir}";
|
||||
};
|
||||
|
||||
cratesDeps = craneLib.buildDepsOnly (common // { src = mkSrc { dirs = [ ]; }; });
|
||||
|
||||
mkWorkspaceLibsSrc =
|
||||
let
|
||||
base = mkSrc {
|
||||
dirs = workspaceDeps;
|
||||
inherit assets;
|
||||
};
|
||||
dummyCrate = craneLib.mkDummySrc { src = cratesRoot + "/${dir}"; };
|
||||
in
|
||||
pkgs.runCommandLocal "source" { } ''
|
||||
cp -r ${base} $out
|
||||
chmod -R u+w $out
|
||||
rm -rf "$out/${crateDir}/${dir}"
|
||||
cp -r ${dummyCrate} "$out/${crateDir}/${dir}"
|
||||
'';
|
||||
|
||||
workspaceLibs =
|
||||
if workspaceDeps == [ ] then
|
||||
cratesDeps
|
||||
else
|
||||
craneLib.buildPackage (
|
||||
common
|
||||
// {
|
||||
pname = "${dir}-workspace-libs";
|
||||
src = mkWorkspaceLibsSrc;
|
||||
cargoArtifacts = cratesDeps;
|
||||
doInstallCargoArtifacts = true;
|
||||
postInstall = ''
|
||||
cargo clean --release -p ${dir}
|
||||
'';
|
||||
}
|
||||
);
|
||||
in
|
||||
craneLib.buildPackage (
|
||||
common
|
||||
// {
|
||||
src = mkSrc {
|
||||
dirs = closure;
|
||||
inherit assets;
|
||||
};
|
||||
cargoArtifacts = workspaceLibs;
|
||||
}
|
||||
);
|
||||
in
|
||||
{
|
||||
inherit buildWorkspaceCrate;
|
||||
}
|
||||
Reference in New Issue
Block a user