From 7914fcc06fc3ecb3ecde3131ea469d84a84ff4a4 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Tue, 7 Oct 2025 12:42:32 -0700 Subject: [PATCH 01/39] Use BSR generate SpongeBob client --- .cargo/config.toml | 3 + Cargo.lock | 244 ++++++++++++++++++++++----- crates/spongebob/Cargo.toml | 6 +- crates/spongebob/build.rs | 5 - crates/spongebob/proto/service.proto | 28 --- crates/spongebob/src/lib.rs | 32 +--- 6 files changed, 215 insertions(+), 103 deletions(-) create mode 100644 .cargo/config.toml delete mode 100644 crates/spongebob/build.rs delete mode 100644 crates/spongebob/proto/service.proto diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000000..f7cf7de3c8 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,3 @@ +[registries.buf] +index = "sparse+https://buf.build/gen/cargo/" +credential-provider = "cargo:token" diff --git a/Cargo.lock b/Cargo.lock index 8912efdc1a..64ea5087ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,6 +130,28 @@ dependencies = [ "term", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -155,10 +177,11 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "axum" -version = "0.8.6" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ + "async-trait", "axum-core", "bytes", "futures-util", @@ -171,26 +194,29 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "serde_core", + "rustversion", + "serde", "sync_wrapper", - "tower", + "tower 0.5.2", "tower-layer", "tower-service", ] [[package]] name = "axum-core" -version = "0.5.5" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ + "async-trait", "bytes", - "futures-core", + "futures-util", "http", "http-body", "http-body-util", "mime", "pin-project-lite", + "rustversion", "sync_wrapper", "tower-layer", "tower-service", @@ -1160,7 +1186,7 @@ dependencies = [ "google-cloud-wkt", "http", "pin-project", - "rand", + "rand 0.9.2", "serde", "serde_json", "thiserror 2.0.17", @@ -1181,15 +1207,15 @@ dependencies = [ "http", "http-body-util", "percent-encoding", - "prost", - "prost-types", + "prost 0.14.1", + "prost-types 0.14.1", "reqwest", "rustc_version", "serde", "serde_json", "thiserror 2.0.17", "tokio", - "tonic", + "tonic 0.14.2", "tonic-prost", ] @@ -1287,8 +1313,8 @@ dependencies = [ "md5", "percent-encoding", "pin-project", - "prost", - "prost-types", + "prost 0.14.1", + "prost-types 0.14.1", "reqwest", "serde", "serde_json", @@ -1296,7 +1322,7 @@ dependencies = [ "sha2", "thiserror 2.0.17", "tokio", - "tonic", + "tonic 0.14.2", "tracing", "uuid", ] @@ -1582,7 +1608,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.0", "system-configuration", "tokio", "tower-service", @@ -2081,9 +2107,9 @@ dependencies = [ [[package]] name = "matchit" -version = "0.8.4" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "md-5" @@ -2169,6 +2195,26 @@ dependencies = [ "zstd", ] +[[package]] +name = "minimal_spongebob_community_neoeinstein-prost" +version = "0.4.0-20251007185801-98893cf03142.1" +source = "sparse+https://buf.build/gen/cargo/" +checksum = "0abe641c48e94dee645ebec4acf117dcbe3d77648e43f0d91c16ea41754a031d" +dependencies = [ + "prost 0.13.5", + "prost-types 0.13.5", +] + +[[package]] +name = "minimal_spongebob_community_neoeinstein-tonic" +version = "0.4.1-20251007185801-98893cf03142.1" +source = "sparse+https://buf.build/gen/cargo/" +checksum = "c166b27b766a89f4dd619b7215d6613cc80af29e6a3f274b568624a298b60da4" +dependencies = [ + "minimal_spongebob_community_neoeinstein-prost", + "tonic 0.12.3", +] + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2716,6 +2762,16 @@ dependencies = [ "hex", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive 0.13.5", +] + [[package]] name = "prost" version = "0.14.1" @@ -2723,7 +2779,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.14.1", ] [[package]] @@ -2739,8 +2795,8 @@ dependencies = [ "once_cell", "petgraph", "prettyplease", - "prost", - "prost-types", + "prost 0.14.1", + "prost-types 0.14.1", "pulldown-cmark", "pulldown-cmark-to-cmark", "regex", @@ -2748,6 +2804,19 @@ dependencies = [ "tempfile", ] +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.11.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "prost-derive" version = "0.14.1" @@ -2761,13 +2830,22 @@ dependencies = [ "syn", ] +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost 0.13.5", +] + [[package]] name = "prost-types" version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" dependencies = [ - "prost", + "prost 0.14.1", ] [[package]] @@ -2803,7 +2881,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2", + "socket2 0.6.0", "thiserror 2.0.17", "tokio", "tracing", @@ -2819,7 +2897,7 @@ dependencies = [ "bytes", "getrandom 0.3.3", "lru-slab", - "rand", + "rand 0.9.2", "ring", "rustc-hash", "rustls", @@ -2840,7 +2918,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.0", "tracing", "windows-sys 0.60.2", ] @@ -2860,14 +2938,35 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ - "rand_chacha", - "rand_core", + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", ] [[package]] @@ -2877,7 +2976,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", ] [[package]] @@ -3015,7 +3123,7 @@ dependencies = [ "tokio-native-tls", "tokio-rustls", "tokio-util", - "tower", + "tower 0.5.2", "tower-http", "tower-service", "url", @@ -3453,6 +3561,16 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.0" @@ -3467,10 +3585,10 @@ dependencies = [ name = "spongebob" version = "0.1.0" dependencies = [ - "prost", + "minimal_spongebob_community_neoeinstein-prost", + "minimal_spongebob_community_neoeinstein-tonic", "thiserror 2.0.17", - "tonic", - "tonic-prost", + "tonic 0.12.3", "tonic-prost-build", "tracing", "uuid", @@ -3759,7 +3877,7 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "socket2", + "socket2 0.6.0", "tokio-macros", "windows-sys 0.59.0", ] @@ -3873,10 +3991,11 @@ checksum = "d163a63c116ce562a22cda521fcc4d79152e7aba014456fb5eb442f6d6a10109" [[package]] name = "tonic" -version = "0.14.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ + "async-stream", "async-trait", "axum", "base64 0.22.1", @@ -3890,13 +4009,38 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", + "prost 0.13.5", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" +dependencies = [ + "base64 0.22.1", + "bytes", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", "rustls-native-certs", - "socket2", "sync_wrapper", "tokio", "tokio-rustls", "tokio-stream", - "tower", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -3921,8 +4065,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" dependencies = [ "bytes", - "prost", - "tonic", + "prost 0.14.1", + "tonic 0.14.2", ] [[package]] @@ -3934,7 +4078,7 @@ dependencies = [ "prettyplease", "proc-macro2", "prost-build", - "prost-types", + "prost-types 0.14.1", "quote", "syn", "tempfile", @@ -3994,6 +4138,26 @@ dependencies = [ "web-sys", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.2" @@ -4026,7 +4190,7 @@ dependencies = [ "http-body", "iri-string", "pin-project-lite", - "tower", + "tower 0.5.2", "tower-layer", "tower-service", ] diff --git a/crates/spongebob/Cargo.toml b/crates/spongebob/Cargo.toml index eab4f6075f..c389d8f9f6 100644 --- a/crates/spongebob/Cargo.toml +++ b/crates/spongebob/Cargo.toml @@ -5,10 +5,10 @@ edition.workspace = true publish.workspace = true [dependencies] -prost.workspace = true +minimal_spongebob_community_neoeinstein-prost = { version = "0.4.0-20251007185801-98893cf03142.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-tonic = { version = "0.4.1-20251007185801-98893cf03142.1", registry = "buf" } thiserror.workspace = true -tonic-prost.workspace = true -tonic.workspace = true +tonic = "0.12.3" tracing.workspace = true uuid.workspace = true diff --git a/crates/spongebob/build.rs b/crates/spongebob/build.rs deleted file mode 100644 index 042c7afe08..0000000000 --- a/crates/spongebob/build.rs +++ /dev/null @@ -1,5 +0,0 @@ -fn main() { - tonic_prost_build::configure() - .compile_protos(&["proto/service.proto"], &["proto"]) - .unwrap(); -} diff --git a/crates/spongebob/proto/service.proto b/crates/spongebob/proto/service.proto deleted file mode 100644 index 0657262e2b..0000000000 --- a/crates/spongebob/proto/service.proto +++ /dev/null @@ -1,28 +0,0 @@ -syntax = "proto3"; - -package spongebob.v1; - -service SpongeBobService { - rpc CreateInvocation(CreateInvocationRequest) returns (Invocation) {} - rpc CreateFile(CreateFileRequest) returns (File) {} -} - -message Invocation { - string name = 1; -} - -message CreateInvocationRequest { - string invocation_id = 1; - Invocation invocation = 2; -} - -message File { - string name = 1; - bytes contents = 2; -} - -message CreateFileRequest { - string parent = 1; - string file_id = 2; - File file = 3; -} diff --git a/crates/spongebob/src/lib.rs b/crates/spongebob/src/lib.rs index bbe7dbbdc6..e079e283dd 100644 --- a/crates/spongebob/src/lib.rs +++ b/crates/spongebob/src/lib.rs @@ -1,15 +1,8 @@ -use tonic::transport::{Channel, ClientTlsConfig}; +use minimal_spongebob_community_neoeinstein_prost::spongebob::v1::{CreateFileRequest, CreateInvocationRequest, File, Invocation}; +use minimal_spongebob_community_neoeinstein_tonic::spongebob::v1::tonic::sponge_bob_service_client::SpongeBobServiceClient; +use tonic::transport::Channel; use uuid::Uuid; -use crate::api::{ - CreateFileRequest, CreateInvocationRequest, File, Invocation, - sponge_bob_service_client::SpongeBobServiceClient, -}; - -pub mod api { - tonic::include_proto!("spongebob.v1"); -} - const ENDPOINT: &str = "https://spongebob.minimal.farm"; #[derive(thiserror::Error, Debug)] @@ -18,8 +11,6 @@ pub enum SpongeBobError { Connection(#[from] tonic::transport::Error), #[error("SpongeBob service error: {0}")] Service(#[from] tonic::Status), - #[error("I/O error: {0}")] - Io(#[from] std::io::Error), } pub type Result = std::result::Result; @@ -45,7 +36,7 @@ impl SpongeBobInvocation { file_id, file: Some(File { name: file_name.to_string(), - contents, + contents: contents.into(), }), }; @@ -64,20 +55,7 @@ impl SpongeBobInvocation { impl SpongeBob { pub async fn new() -> Result { - let tls_config = ClientTlsConfig::new().with_native_roots(); - let channel = Channel::from_static(ENDPOINT) - .tls_config(tls_config) - .unwrap() - .connect() - .await?; - let service = SpongeBobServiceClient::new(channel); - - Ok(SpongeBob { service }) - } - - pub async fn with_endpoint(endpoint: &'static str) -> Result { - let channel = Channel::from_static(endpoint).connect().await?; - let service = SpongeBobServiceClient::new(channel); + let service = SpongeBobServiceClient::connect(ENDPOINT).await?; Ok(SpongeBob { service }) } From 5fecb9573a28c90b7ae0d3b24b8393892ec3ec56 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Wed, 8 Oct 2025 15:25:10 -0700 Subject: [PATCH 02/39] build event vibes --- BUILD_EVENTS_INTEGRATION.md | 331 +++++++++++++++++ BUILD_EVENT_DESIGN.md | 339 ++++++++++++++++++ Cargo.lock | 132 +++++-- Cargo.toml | 4 + build_events.jsonl | 8 + crates/build-events-proto/Cargo.toml | 35 ++ crates/build-events-proto/build.rs | 9 + crates/build-events-proto/proto/events.proto | 91 +++++ crates/build-events-proto/src/convert.rs | 337 +++++++++++++++++ crates/build-events-proto/src/lib.rs | 136 +++++++ crates/build-events-proto/src/service.rs | 213 +++++++++++ .../src/spongebob_convert.rs | 165 +++++++++ .../src/spongebob_subscriber_v2.rs | 112 ++++++ crates/build-events-proto/src/subscriber.rs | 156 ++++++++ crates/build-events/Cargo.toml | 20 ++ crates/build-events/examples/basic_usage.rs | 169 +++++++++ crates/build-events/src/bus.rs | 235 ++++++++++++ crates/build-events/src/dispatcher.rs | 307 ++++++++++++++++ crates/build-events/src/events.rs | 175 +++++++++ crates/build-events/src/lib.rs | 130 +++++++ crates/build-events/src/subscriber.rs | 126 +++++++ .../build-events/src/subscribers/json_file.rs | 204 +++++++++++ crates/build-events/src/subscribers/logger.rs | 165 +++++++++ crates/build-events/src/subscribers/mod.rs | 11 + crates/build-events/tests/integration_test.rs | 232 ++++++++++++ crates/minimal/Cargo.toml | 2 + crates/minimal/src/cmd_build.rs | 77 ++-- .../minimal/src/cmd_unsafe_patched_build.rs | 14 +- crates/spongebob/Cargo.toml | 9 +- crates/spongebob/src/lib.rs | 115 +++--- 30 files changed, 3953 insertions(+), 106 deletions(-) create mode 100644 BUILD_EVENTS_INTEGRATION.md create mode 100644 BUILD_EVENT_DESIGN.md create mode 100644 build_events.jsonl create mode 100644 crates/build-events-proto/Cargo.toml create mode 100644 crates/build-events-proto/build.rs create mode 100644 crates/build-events-proto/proto/events.proto create mode 100644 crates/build-events-proto/src/convert.rs create mode 100644 crates/build-events-proto/src/lib.rs create mode 100644 crates/build-events-proto/src/service.rs create mode 100644 crates/build-events-proto/src/spongebob_convert.rs create mode 100644 crates/build-events-proto/src/spongebob_subscriber_v2.rs create mode 100644 crates/build-events-proto/src/subscriber.rs create mode 100644 crates/build-events/Cargo.toml create mode 100644 crates/build-events/examples/basic_usage.rs create mode 100644 crates/build-events/src/bus.rs create mode 100644 crates/build-events/src/dispatcher.rs create mode 100644 crates/build-events/src/events.rs create mode 100644 crates/build-events/src/lib.rs create mode 100644 crates/build-events/src/subscriber.rs create mode 100644 crates/build-events/src/subscribers/json_file.rs create mode 100644 crates/build-events/src/subscribers/logger.rs create mode 100644 crates/build-events/src/subscribers/mod.rs create mode 100644 crates/build-events/tests/integration_test.rs diff --git a/BUILD_EVENTS_INTEGRATION.md b/BUILD_EVENTS_INTEGRATION.md new file mode 100644 index 0000000000..e761e0cf48 --- /dev/null +++ b/BUILD_EVENTS_INTEGRATION.md @@ -0,0 +1,331 @@ +# Build Events Integration - ✅ COMPLETE + +## Overview +Successfully integrated the build events system from minpkgs into spongebob for persistent storage and remote observability. The integration is now fully wired into the minpkgs CLI and emits build lifecycle events. + +## ✅ What Was Implemented + +### 1. Spongebob Backend (Go) + +#### Database Schema (schema.sql:28-38) +```sql +CREATE TABLE build_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + invocation_id UUID NOT NULL, + event_type VARCHAR(50) NOT NULL, + timestamp_millis BIGINT NOT NULL, + event_data JSONB NOT NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + FOREIGN KEY (invocation_id) REFERENCES invocations(id) ON DELETE CASCADE +); +``` + +With indexes on `invocation_id`, `timestamp_millis`, and `event_type`. + +#### SQL Queries (queries.sql:35-54) +- `CreateBuildEvent` - Insert event with JSONB storage +- `ListBuildEventsByInvocation` - Query by invocation with pagination +- `ListBuildEventsByTimeRange` - Query by timestamp range +- `GetBuildEvent` - Retrieve single event by ID + +#### Protobuf Definitions +**proto/spongebob/v1/build_events.proto** (new file): +- `BuildEvent` with oneof for all event types +- `BuildStarted`, `BuildFinished`, `TargetStarted`, `TargetCompleted`, `ActionStarted`, `ActionCompleted` +- `TargetKind` enum + +**proto/spongebob/v1/service.proto** (extended): +- `PublishBuildEvent(PublishBuildEventRequest) → PublishBuildEventResponse` +- `ListBuildEvents(ListBuildEventsRequest) → ListBuildEventsResponse` +- `StreamBuildEvents(StreamBuildEventsRequest) → stream BuildEvent` + +Published to BSR: `buf.build/minimal/spongebob:302595534f7d` + +#### Go Service Implementation (internal/server/grpc.go) + +**PublishBuildEvent RPC** (grpc.go:319-374): +- Validates invocation exists +- Extracts event type and timestamp +- Marshals to JSON for storage +- Broadcasts to streaming clients +- Returns resource name: `invocations/{id}/events/{event-id}` + +**ListBuildEvents RPC** (grpc.go:376-448): +- Paginated event listing (default 50, max 100) +- Base64-encoded page tokens +- Unmarshals JSONB to proto +- Returns events with resource names + +**StreamBuildEvents RPC** (grpc.go:450-507): +- Real-time server-side streaming +- Optional invocation_id and timestamp filtering +- In-memory broadcast to all connected clients +- Non-blocking sends (skips slow clients) + +**Helper Functions** (grpc.go:583-651): +- `getEventTypeAndTimestamp()` - Extract from proto oneofs +- `getEventInvocationID()` - Extract invocation from events +- `broadcastBuildEvent()` - Broadcast to stream subscribers +- `generateClientID()` - Unique stream client IDs +- `encodePageToken()` / `decodePageToken()` - Pagination + +### 2. Minpkgs Client (Rust) + +#### Build Events Proto Crate + +**spongebob_convert.rs** (feature-gated): +- `to_proto_build_event()` - Main conversion function +- Separate converters for each event type +- Converts Rust enums to proto i32 values +- Zero-copy where possible, clones only when needed + +**spongebob_subscriber_v2.rs** (feature-gated): +```rust +pub struct SpongeBobSubscriberV2 { + invocation: Arc>, +} + +impl BuildEventSubscriber for SpongeBobSubscriberV2 { + async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { + let proto_event = to_proto_build_event(event); + let mut invocation = self.invocation.lock().await; + invocation.publish_build_event(proto_event).await?; + Ok(()) + } +} +``` + +**Feature Flags** (Cargo.toml): +- `spongebob-subscriber` feature gates spongebob integration +- Optional dependency on `spongebob` crate +- Optional BSR proto import for conversion + +#### Spongebob Crate Updates + +**lib.rs:50-85** - New method on `SpongeBobInvocation`: +```rust +pub async fn publish_build_event(&mut self, event: BuildEvent) -> Result<()> { + let request = PublishBuildEventRequest { + parent: self.resource_name.clone(), + event: Some(event), + }; + self.service.publish_build_event(request).await?; + Ok(()) +} +``` + +## 🏗️ Architecture + +``` +┌─────────────────────────────────────────────────────┐ +│ Minpkgs Build │ +│ │ +│ BuildEventBus → BuildEventDispatcher │ +│ ↓ │ +│ SpongeBobSubscriberV2 │ +└─────────────────────┬───────────────────────────────┘ + │ gRPC (PublishBuildEvent) + ↓ +┌─────────────────────────────────────────────────────┐ +│ Spongebob Service (Go) │ +│ │ +│ PublishBuildEvent RPC │ +│ ↓ │ +│ PostgreSQL (JSONB storage) │ +│ ↓ │ +│ Broadcast to streaming clients │ +│ │ +│ ListBuildEvents RPC ←── dash/API clients │ +│ StreamBuildEvents RPC ←── real-time UI │ +└─────────────────────────────────────────────────────┘ +``` + +## 📝 Usage Example + +### In minpkgs build: + +```rust +use build_events::{BuildEventBus, BuildEventDispatcher}; +use build_events_proto::SpongeBobSubscriberV2; + +#[tokio::main] +async fn main() -> Result<()> { + // Connect to spongebob + let mut spongebob = spongebob::SpongeBob::new().await?; + let invocation = spongebob.create_invocation("my-build").await?; + + println!("Build logs: {}", invocation.url()); + + // Setup build events + let event_bus = BuildEventBus::new(10000); + + let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); + dispatcher.add_subscriber(Box::new( + SpongeBobSubscriberV2::from_invocation(invocation) + )); + + tokio::spawn(async move { + dispatcher.run().await; + }); + + // Events are automatically published to spongebob + event_bus.emit(BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command_line: vec!["cargo".to_string(), "build".to_string()], + timestamp_millis: current_millis(), + working_directory: env::current_dir()?.to_string_lossy().to_string(), + })); + + // ... run build ... + + Ok(()) +} +``` + +## ✅ Testing + +All components build successfully: +- ✅ Spongebob Go service: `go build -o spongebob .` +- ✅ Minpkgs workspace: `cargo build` +- ✅ Build-events-proto with spongebob-subscriber: `cargo build --features spongebob-subscriber` +- ✅ Minpkgs CLI with build events: `cargo build --package minimal` + +To test the end-to-end integration: +```bash +cd minpkgs +cargo run -- build --package bash +# Events will be automatically published to spongebob.minimal.farm +# Check the invocation URL printed at the end +``` + +### 3. Minpkgs CLI Integration + +#### Updated Files +**crates/minimal/Cargo.toml** (lines 10-11): +- Added `build-events.workspace = true` +- Added `build-events-proto = { path = "../build-events-proto", features = ["spongebob-subscriber"] }` + +**crates/minimal/src/cmd_build.rs**: + +**Imports** (lines 4-6): +```rust +use build_events::events::{BuildEvent, BuildFinished, BuildStarted, current_millis}; +use build_events::{BuildEventBus, BuildEventDispatcher}; +use build_events_proto::SpongeBobSubscriberV2; +``` + +**Build Events Setup** (lines 74-110): +```rust +// Setup build events system +let event_bus = BuildEventBus::new(10000); + +// Create dispatcher with SpongeBobSubscriberV2 if we have an invocation +if let Some(ref invocation) = spongebob_invocation { + let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); + let subscriber = SpongeBobSubscriberV2::from_invocation(invocation.clone()); + dispatcher.add_subscriber(Box::new(subscriber)); + + // Spawn dispatcher in background + tokio::spawn(async move { + dispatcher.run().await; + }); +} + +// Get invocation_id for events +let invocation_id = spongebob_invocation + .as_ref() + .map(|inv| inv.resource_name().to_string()) + .unwrap_or_else(|| "local".to_string()); + +// Get command line for BuildStarted event +let command_line = std::env::args().collect::>(); +let working_directory = std::env::current_dir() + .unwrap_or_default() + .to_string_lossy() + .to_string(); + +// Emit BuildStarted event +event_bus.emit(BuildEvent::BuildStarted(BuildStarted { + invocation_id: invocation_id.clone(), + command_line, + timestamp_millis: current_millis(), + working_directory, +})); +``` + +**Build Execution with Event Emission** (lines 153-166): +```rust +// Determine if build was successful +let build_succeeded = build_success.is_ok(); +let error_message = build_success.as_ref().err().map(|e| e.to_string()); + +// Emit BuildFinished event +event_bus.emit(BuildEvent::BuildFinished(BuildFinished { + invocation_id: invocation_id.clone(), + success: build_succeeded, + timestamp_millis: current_millis(), + error_message, +})); + +// Propagate error if build failed +build_success.context("Failed to execute build")?; +``` + +## 🚀 Deployment Steps + +To deploy this integration: + +1. **Deploy Database Migration** - Run schema.sql on production database + +2. **Deploy Spongebob Service** - Deploy updated Go service with new RPCs + +3. **Deploy Minpkgs Binary** - Build and deploy minpkgs with the new integration: + ```bash + cd minpkgs + cargo build --release + ``` + +4. **Test End-to-End**: + - Run minpkgs build with spongebob integration + - Verify events in PostgreSQL: `SELECT * FROM build_events;` + - Test ListBuildEvents API + - Test StreamBuildEvents with grpcurl or dash UI + +## 📦 Files Modified + +### Spongebob (Go) +- `schema.sql` - Added build_events table +- `queries.sql` - Added 4 queries +- `proto/spongebob/v1/build_events.proto` - New file +- `proto/spongebob/v1/service.proto` - Added 3 RPCs +- `internal/server/grpc.go` - ~350 lines of implementation +- `internal/db/models.go` - Generated BuildEvent model +- `internal/db/queries.sql.go` - Generated query functions + +### Minpkgs (Rust) +- `crates/build-events-proto/Cargo.toml` - Added spongebob-subscriber feature +- `crates/build-events-proto/src/lib.rs` - Feature-gated exports +- `crates/build-events-proto/src/spongebob_convert.rs` - New file +- `crates/build-events-proto/src/spongebob_subscriber_v2.rs` - New file +- `crates/spongebob/Cargo.toml` - Updated to latest proto +- `crates/spongebob/src/lib.rs` - Added publish_build_event method +- `crates/minimal/Cargo.toml` - Added build-events and build-events-proto dependencies +- `crates/minimal/src/cmd_build.rs` - Wired BuildEventBus, dispatcher, and event emission + +## 🎯 Key Design Decisions + +1. **JSONB Storage** - Flexible schema evolution without migrations +2. **Event Type Column** - Enables fast filtering without parsing JSON +3. **Timestamp Index** - Supports time-range queries for analytics +4. **Non-blocking Broadcasts** - Slow stream clients don't affect ingestion +5. **Feature Flag** - Optional spongebob integration doesn't force dependency +6. **Arc>** - Thread-safe sharing for async subscriber + +## 🔒 Production Considerations + +- **Rate Limiting** - Consider limiting PublishBuildEvent requests per invocation +- **Event Size Limits** - Current max JSONB size should be documented +- **Stream Client Limits** - Monitor number of concurrent StreamBuildEvents connections +- **Database Performance** - Monitor JSONB query performance at scale +- **Error Handling** - Subscriber logs warnings but doesn't fail builds on errors diff --git a/BUILD_EVENT_DESIGN.md b/BUILD_EVENT_DESIGN.md new file mode 100644 index 0000000000..8bec45dc6c --- /dev/null +++ b/BUILD_EVENT_DESIGN.md @@ -0,0 +1,339 @@ +# Build Events System Design Proposal (Revised) + +## Overview +Create a dedicated build event protocol (BEP) system for `minimal` that enables remote observability of build execution. This system will be separate from diagnostic tracing and will emit structured, strongly-typed events that can be consumed by local and remote services. + +## Goals +- Separate build events from diagnostic logs (`tracing`) +- Provide structured, serializable Rust events +- Enable multiple concurrent subscribers (local UI, remote services, file writers) +- Maintain low overhead and back-pressure handling +- Allow subscribers to choose their own serialization format + +## Crate Structure + +**Crate name**: `build-events` + +**Location**: `crates/build-events/` + +**Dependencies**: +```toml +[dependencies] +tokio = { version = "1", features = ["sync", "time"] } +serde = { version = "1", features = ["derive"] } +thiserror = "1" +async-trait = "0.1" + +[dev-dependencies] +tokio = { version = "1", features = ["full", "test-util"] } +serde_json = "1" +``` + +## Architecture + +### Core Components + +1. **Event Types** (Pure Rust) + - Strongly-typed event enums representing build lifecycle + - Derive `Clone`, `Debug`, `Serialize`, `Deserialize` + - Minimal initial set: BuildStarted, TargetStarted, TargetCompleted, ActionStarted, ActionCompleted, BuildFinished + +2. **BuildEventBus** (publisher) + - Central event dispatcher using `tokio::sync::broadcast` + - Thread-safe, can be cloned and passed to any component + - Non-blocking emission with configurable buffer size + +3. **BuildEventSubscriber** (trait) + - Similar to `tracing::Subscriber` + - Allows pluggable event consumers + - Async trait for handling events + - **Subscribers handle their own serialization** (JSON, protobuf, bincode, etc.) + +4. **Event Stream Handler** + - Manages subscriber registration + - Distributes events from broadcast channel to subscribers + - Handles subscriber lifecycle + +## API Design + +### Event Emission (Producer Side) + +```rust +// Initialize at build start +let event_bus = BuildEventBus::new(capacity: 10000); + +// Clone and use anywhere in the codebase +let bus = event_bus.clone(); +bus.emit(BuildEvent::BuildStarted { + invocation_id: uuid::Uuid::new_v4().to_string(), + command_line: args, + timestamp_millis: current_millis(), + working_directory: env::current_dir().unwrap(), +}); + +// Emit from any component +bus.emit(BuildEvent::TargetStarted { + label: "//foo:bar".to_string(), + target_kind: TargetKind::Binary, + timestamp_millis: current_millis(), +}); +``` + +### Event Consumption (Consumer Side) + +```rust +#[async_trait] +pub trait BuildEventSubscriber: Send + Sync { + async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError>; + + fn name(&self) -> &str; + + // Optional: called when subscriber is dropped + async fn on_close(&self) -> Result<(), SubscriberError> { + Ok(()) + } +} + +// Register subscribers +let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); +dispatcher.add_subscriber(Box::new(JsonFileWriter::new("build_events.jsonl"))); +dispatcher.add_subscriber(Box::new(TerminalUI::new())); + +// Start processing (runs until bus is closed) +tokio::spawn(async move { + dispatcher.run().await; +}); +``` + +## Event Schema (Pure Rust) + +```rust +use serde::{Serialize, Deserialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum BuildEvent { + BuildStarted(BuildStarted), + BuildFinished(BuildFinished), + TargetStarted(TargetStarted), + TargetCompleted(TargetCompleted), + ActionStarted(ActionStarted), + ActionCompleted(ActionCompleted), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct BuildStarted { + pub invocation_id: String, + pub command_line: Vec, + pub timestamp_millis: i64, + pub working_directory: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct BuildFinished { + pub invocation_id: String, + pub success: bool, + pub timestamp_millis: i64, + pub error_message: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TargetStarted { + pub label: String, + pub target_kind: TargetKind, + pub timestamp_millis: i64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TargetCompleted { + pub label: String, + pub success: bool, + pub timestamp_millis: i64, + pub error_message: Option, + pub outputs: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ActionStarted { + pub action_id: String, + pub label: String, + pub mnemonic: String, + pub timestamp_millis: i64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ActionCompleted { + pub action_id: String, + pub success: bool, + pub timestamp_millis: i64, + pub exit_code: i32, + pub stdout: Option, + pub stderr: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TargetKind { + Binary, + Library, + Test, +} +``` + +## Module Structure + +``` +crates/build-events/ +├── Cargo.toml +└── src/ + ├── lib.rs # Public API exports + ├── bus.rs # BuildEventBus implementation + ├── subscriber.rs # BuildEventSubscriber trait + ├── dispatcher.rs # Event distribution logic + ├── events.rs # Event type definitions + └── subscribers/ # Built-in subscriber implementations + ├── mod.rs + ├── logger.rs # Logs events via tracing (for debugging) + └── json_file.rs # Writes JSON-lines to file +``` + +## Future: Protobuf Subscriber Layer + +When you need gRPC support, create a separate crate: + +``` +crates/build-events-proto/ +├── Cargo.toml +├── build.rs # prost-build configuration +├── proto/ +│ └── events.proto # Protobuf definitions +└── src/ + ├── lib.rs + ├── convert.rs # Conversions: BuildEvent <-> proto + └── subscriber.rs # GrpcStreamSubscriber implementation +``` + +The proto crate would: +- Define protobuf schema +- Implement `From` for proto types +- Provide a `GrpcStreamSubscriber` that converts and streams events + +## Integration with Minimal + +### 1. Workspace Configuration +Add to root `Cargo.toml`: +```toml +[workspace] +members = [ + "crates/build-events", + # ... other crates +] +``` + +### 2. Usage in Main Binary + +```rust +use build_events::{BuildEventBus, BuildEventDispatcher, BuildEvent, subscribers}; + +#[tokio::main] +async fn main() { + // Initialize tracing (existing diagnostic logs) + tracing_subscriber::fmt::init(); + + // Initialize build events + let event_bus = BuildEventBus::new(10000); + + // Setup subscribers + let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); + dispatcher.add_subscriber(Box::new( + subscribers::JsonFileWriter::new("build_events.jsonl").unwrap() + )); + + // Start event processing + let dispatcher_handle = tokio::spawn(async move { + dispatcher.run().await + }); + + // Pass event_bus to build executor + let build_result = run_build(event_bus.clone(), build_config).await; + + // Shutdown: drop event_bus, wait for dispatcher to drain + drop(event_bus); + let _ = dispatcher_handle.await; + + // Continue with result handling... +} +``` + +### 3. Emitting Events in Build Logic + +```rust +async fn execute_target(bus: BuildEventBus, target: &Target) -> Result<()> { + bus.emit(BuildEvent::TargetStarted(TargetStarted { + label: target.label.clone(), + target_kind: target.kind, + timestamp_millis: current_millis(), + })); + + // ... build logic ... + + bus.emit(BuildEvent::TargetCompleted(TargetCompleted { + label: target.label.clone(), + success: result.is_ok(), + timestamp_millis: current_millis(), + error_message: result.as_ref().err().map(|e| e.to_string()), + outputs: result.as_ref().ok().cloned().unwrap_or_default(), + })); + + result +} +``` + +## Implementation Phases + +### Phase 1: Core Infrastructure +- [ ] Setup `build-events` crate with serde +- [ ] Define Rust event types (6 event variants) +- [ ] Implement `BuildEventBus` with broadcast channel +- [ ] Define `BuildEventSubscriber` trait +- [ ] Implement `BuildEventDispatcher` +- [ ] Write unit tests for bus and dispatcher + +### Phase 2: Built-in Subscribers +- [ ] Implement `LoggerSubscriber` (emits to tracing for debugging) +- [ ] Implement `JsonFileWriter` (JSON-lines format) +- [ ] Add integration tests + +### Phase 3: Integration +- [ ] Add to `minimal` workspace +- [ ] Wire into main binary +- [ ] Add event emission to key build phases +- [ ] Documentation and examples + +### Phase 4: Future (Out of Scope) +- `build-events-proto` crate with protobuf definitions +- gRPC streaming subscriber +- Event filtering/sampling +- Additional event types +- Metrics/analytics subscriber + +## Error Handling + +- **Bus emission**: Best-effort, never blocks. If channel is full, events are dropped (lagging subscribers) +- **Subscriber errors**: Logged via `tracing::warn!`, but don't stop other subscribers +- **Backpressure**: Broadcast channel size is configurable; slow subscribers will lag and miss events + +## Testing Strategy + +- Unit tests for event bus mechanics +- Unit tests for dispatcher subscription management +- Integration tests with mock subscribers +- Benchmark tests for overhead measurement +- Example subscriber implementations as test fixtures + +## Open Questions / Future Considerations + +1. Should we add event sequence numbers for ordering guarantees? +2. Do we need event filtering at the bus level? +3. Should the JSON file writer support rotation/compression? +4. Metrics on dropped events? diff --git a/Cargo.lock b/Cargo.lock index 64ea5087ca..619251f37e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,14 +182,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", - "axum-core", + "axum-core 0.4.5", "bytes", "futures-util", "http", "http-body", "http-body-util", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", @@ -202,6 +202,31 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871" +dependencies = [ + "axum-core 0.5.5", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.4.5" @@ -222,6 +247,24 @@ dependencies = [ "tower-service", ] +[[package]] +name = "axum-core" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.76" @@ -365,6 +408,40 @@ dependencies = [ "serde", ] +[[package]] +name = "build-events" +version = "0.0.1" +dependencies = [ + "async-trait", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.17", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "build-events-proto" +version = "0.0.1" +dependencies = [ + "async-trait", + "build-events", + "minimal_spongebob_community_neoeinstein-prost", + "prost 0.14.1", + "spongebob", + "tempfile", + "thiserror 2.0.17", + "tokio", + "tokio-stream", + "tonic 0.14.2", + "tonic-prost", + "tonic-prost-build", + "tracing", + "tracing-subscriber", +] + [[package]] name = "build-sandbox" version = "0.1.0" @@ -832,7 +909,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -906,7 +983,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1424,12 +1501,6 @@ dependencies = [ "foldhash", ] -[[package]] -name = "hashbrown" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" - [[package]] name = "hashlink" version = "0.10.0" @@ -1780,7 +1851,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown 0.16.0", + "hashbrown 0.15.5", "serde", "serde_core", ] @@ -2111,6 +2182,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "md-5" version = "0.10.6" @@ -2155,6 +2232,8 @@ version = "0.1.0" dependencies = [ "anyhow", "blake3", + "build-events", + "build-events-proto", "build-sandbox", "bytes", "cache", @@ -2197,9 +2276,9 @@ dependencies = [ [[package]] name = "minimal_spongebob_community_neoeinstein-prost" -version = "0.4.0-20251007185801-98893cf03142.1" +version = "0.4.0-20251008233118-4a2c40ff6cb8.1" source = "sparse+https://buf.build/gen/cargo/" -checksum = "0abe641c48e94dee645ebec4acf117dcbe3d77648e43f0d91c16ea41754a031d" +checksum = "3b7f67266fb3637565abb8b4513f7ac5b6b1db85ef32b52378490b32a5f1cf6d" dependencies = [ "prost 0.13.5", "prost-types 0.13.5", @@ -2207,9 +2286,9 @@ dependencies = [ [[package]] name = "minimal_spongebob_community_neoeinstein-tonic" -version = "0.4.1-20251007185801-98893cf03142.1" +version = "0.4.1-20251008233118-4a2c40ff6cb8.1" source = "sparse+https://buf.build/gen/cargo/" -checksum = "c166b27b766a89f4dd619b7215d6613cc80af29e6a3f274b568624a298b60da4" +checksum = "243b90d6da5fd10bfd4ce7da52788eece61c44d8d5847ca2d0346d435b7ea8af" dependencies = [ "minimal_spongebob_community_neoeinstein-prost", "tonic 0.12.3", @@ -2789,7 +2868,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" dependencies = [ "heck 0.5.0", - "itertools 0.14.0", + "itertools 0.11.0", "log", "multimap", "once_cell", @@ -2824,7 +2903,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn", @@ -3179,7 +3258,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3589,7 +3668,6 @@ dependencies = [ "minimal_spongebob_community_neoeinstein-tonic", "thiserror 2.0.17", "tonic 0.12.3", - "tonic-prost-build", "tracing", "uuid", ] @@ -3736,7 +3814,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3745,7 +3823,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2111ef44dae28680ae9752bb89409e7310ca33a8c621ebe7b106cf5c928b3ac0" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -3922,6 +4000,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] @@ -3997,7 +4076,7 @@ checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ "async-stream", "async-trait", - "axum", + "axum 0.7.9", "base64 0.22.1", "bytes", "h2", @@ -4010,8 +4089,11 @@ dependencies = [ "percent-encoding", "pin-project", "prost 0.13.5", + "rustls-native-certs", + "rustls-pemfile", "socket2 0.5.10", "tokio", + "tokio-rustls", "tokio-stream", "tower 0.4.13", "tower-layer", @@ -4025,8 +4107,11 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" dependencies = [ + "async-trait", + "axum 0.8.6", "base64 0.22.1", "bytes", + "h2", "http", "http-body", "http-body-util", @@ -4036,6 +4121,7 @@ dependencies = [ "percent-encoding", "pin-project", "rustls-native-certs", + "socket2 0.6.0", "sync_wrapper", "tokio", "tokio-rustls", @@ -4673,7 +4759,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 59da735ca2..e585d1c75c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,7 @@ [workspace] members = [ + "crates/build-events", + "crates/build-events-proto", "crates/build-sandbox", "crates/cache", "crates/common", @@ -77,6 +79,8 @@ zstd = "0.13" globset = "0.4" # our crates +build-events = { version = "0.0.1", path = "crates/build-events" } +build-events-proto = { version = "0.0.1", path = "crates/build-events-proto" } build-sandbox = { path = "crates/build-sandbox" } graph = { version = "0.0.1", path = "crates/graph" } cache = { version = "0.0.1", path = "crates/cache" } diff --git a/build_events.jsonl b/build_events.jsonl new file mode 100644 index 0000000000..d4a7c4cea9 --- /dev/null +++ b/build_events.jsonl @@ -0,0 +1,8 @@ +{"type":"build_started","invocation_id":"demo-invocation-123","command_line":["minimal","build","//example:target"],"timestamp_millis":1759951982261,"working_directory":"/Users/bweeks/Code/minimal/minpkgs"} +{"type":"target_started","label":"//example:target","target_kind":"binary","timestamp_millis":1759951982363} +{"type":"action_started","action_id":"action-compile-1","label":"//example:target","mnemonic":"RustcCompile","timestamp_millis":1759951982466} +{"type":"action_completed","action_id":"action-compile-1","success":true,"timestamp_millis":1759951982667,"exit_code":0,"stdout":"Compiling example v0.1.0","stderr":null} +{"type":"action_started","action_id":"action-link-1","label":"//example:target","mnemonic":"RustcLink","timestamp_millis":1759951982769} +{"type":"action_completed","action_id":"action-link-1","success":true,"timestamp_millis":1759951982922,"exit_code":0,"stdout":"Finished release [optimized] target(s)","stderr":null} +{"type":"target_completed","label":"//example:target","success":true,"timestamp_millis":1759951983024,"error_message":null,"outputs":["target/release/example"]} +{"type":"build_finished","invocation_id":"demo-invocation-123","success":true,"timestamp_millis":1759951983126,"error_message":null} diff --git a/crates/build-events-proto/Cargo.toml b/crates/build-events-proto/Cargo.toml new file mode 100644 index 0000000000..f6d08e3783 --- /dev/null +++ b/crates/build-events-proto/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "build-events-proto" +version = "0.0.1" +edition.workspace = true +rust-version = "1.85" +license = "MIT OR Apache-2.0" +publish.workspace = true + +[dependencies] +build-events.workspace = true +prost.workspace = true +tonic.workspace = true +tonic-prost.workspace = true +tokio.workspace = true +thiserror.workspace = true +tracing.workspace = true +async-trait = "0.1" +tokio-stream = { version = "0.1", features = ["sync"] } + +# Use workspace spongebob crate instead of direct BSR imports +spongebob = { path = "../spongebob", optional = true } + +# Need proto types for conversion when spongebob-subscriber feature is enabled +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251008233118-4a2c40ff6cb8.1", registry = "buf", optional = true } + +[features] +default = [] +spongebob-subscriber = ["spongebob", "minimal_spongebob_community_neoeinstein-prost"] + +[build-dependencies] +tonic-prost-build.workspace = true + +[dev-dependencies] +tempfile.workspace = true +tracing-subscriber.workspace = true diff --git a/crates/build-events-proto/build.rs b/crates/build-events-proto/build.rs new file mode 100644 index 0000000000..8172cf6ca3 --- /dev/null +++ b/crates/build-events-proto/build.rs @@ -0,0 +1,9 @@ +fn main() -> Result<(), Box> { + // Compile protobuf files using tonic-prost-build + tonic_prost_build::compile_protos("proto/events.proto")?; + + // Re-run build script if proto files change + println!("cargo:rerun-if-changed=proto/events.proto"); + + Ok(()) +} diff --git a/crates/build-events-proto/proto/events.proto b/crates/build-events-proto/proto/events.proto new file mode 100644 index 0000000000..40472b8f82 --- /dev/null +++ b/crates/build-events-proto/proto/events.proto @@ -0,0 +1,91 @@ +syntax = "proto3"; + +package build_events.v1; + +// BuildEvent is the top-level event wrapper that contains one of the specific event types +message BuildEvent { + oneof event { + BuildStarted build_started = 1; + BuildFinished build_finished = 2; + TargetStarted target_started = 3; + TargetCompleted target_completed = 4; + ActionStarted action_started = 5; + ActionCompleted action_completed = 6; + } +} + +// BuildStarted is emitted when a build invocation starts +message BuildStarted { + string invocation_id = 1; + repeated string command_line = 2; + int64 timestamp_millis = 3; + string working_directory = 4; +} + +// BuildFinished is emitted when a build invocation completes +message BuildFinished { + string invocation_id = 1; + bool success = 2; + int64 timestamp_millis = 3; + optional string error_message = 4; +} + +// TargetStarted is emitted when a build target starts building +message TargetStarted { + string target_id = 1; + string label = 2; + TargetKind target_kind = 3; + int64 timestamp_millis = 4; +} + +// TargetCompleted is emitted when a build target completes +message TargetCompleted { + string target_id = 1; + string label = 2; + bool success = 3; + int64 timestamp_millis = 4; + optional string error_message = 5; + repeated string outputs = 6; + bool cache_hit = 7; +} + +// ActionStarted is emitted when a build action starts executing +message ActionStarted { + string action_id = 1; + string action_name = 2; + string target_id = 3; + int64 timestamp_millis = 4; +} + +// ActionCompleted is emitted when a build action completes +message ActionCompleted { + string action_id = 1; + bool success = 2; + int64 timestamp_millis = 3; + int32 exit_code = 4; + optional string stdout = 5; + optional string stderr = 6; +} + +// TargetKind represents the type of build target +enum TargetKind { + TARGET_KIND_UNSPECIFIED = 0; + TARGET_KIND_BINARY = 1; + TARGET_KIND_LIBRARY = 2; + TARGET_KIND_TEST = 3; +} + +// BuildEventService provides a streaming interface for build events +service BuildEventService { + // Subscribe to build events for a specific invocation or all invocations + rpc StreamEvents(StreamEventsRequest) returns (stream BuildEvent); +} + +// StreamEventsRequest configures the event stream subscription +message StreamEventsRequest { + // Optional filter to only receive events for a specific invocation + optional string invocation_id = 1; + + // Optional: start streaming from this timestamp (milliseconds since epoch) + optional int64 start_timestamp_millis = 2; +} diff --git a/crates/build-events-proto/src/convert.rs b/crates/build-events-proto/src/convert.rs new file mode 100644 index 0000000000..862aaad61d --- /dev/null +++ b/crates/build-events-proto/src/convert.rs @@ -0,0 +1,337 @@ +//! Type conversions between Rust and protobuf types +//! +//! This module provides `From` implementations for converting between +//! the pure Rust `build_events` types and the generated protobuf types. + +use crate::proto; +use build_events::events; + +// ============================================================================ +// Rust -> Proto conversions +// ============================================================================ + +impl From for proto::BuildEvent { + fn from(event: events::BuildEvent) -> Self { + let event = match event { + events::BuildEvent::BuildStarted(e) => { + proto::build_event::Event::BuildStarted(e.into()) + } + events::BuildEvent::BuildFinished(e) => { + proto::build_event::Event::BuildFinished(e.into()) + } + events::BuildEvent::TargetStarted(e) => { + proto::build_event::Event::TargetStarted(e.into()) + } + events::BuildEvent::TargetCompleted(e) => { + proto::build_event::Event::TargetCompleted(e.into()) + } + events::BuildEvent::ActionStarted(e) => { + proto::build_event::Event::ActionStarted(e.into()) + } + events::BuildEvent::ActionCompleted(e) => { + proto::build_event::Event::ActionCompleted(e.into()) + } + }; + + proto::BuildEvent { event: Some(event) } + } +} + +impl From for proto::BuildStarted { + fn from(event: events::BuildStarted) -> Self { + proto::BuildStarted { + invocation_id: event.invocation_id, + command_line: event.command_line, + timestamp_millis: event.timestamp_millis, + working_directory: event.working_directory, + } + } +} + +impl From for proto::BuildFinished { + fn from(event: events::BuildFinished) -> Self { + proto::BuildFinished { + invocation_id: event.invocation_id, + success: event.success, + timestamp_millis: event.timestamp_millis, + error_message: event.error_message, + } + } +} + +impl From for proto::TargetStarted { + fn from(event: events::TargetStarted) -> Self { + proto::TargetStarted { + target_id: event.target_id, + label: event.label, + target_kind: proto::TargetKind::from(event.target_kind) as i32, + timestamp_millis: event.timestamp_millis, + } + } +} + +impl From for proto::TargetCompleted { + fn from(event: events::TargetCompleted) -> Self { + proto::TargetCompleted { + target_id: event.target_id, + label: event.label, + success: event.success, + timestamp_millis: event.timestamp_millis, + error_message: event.error_message, + outputs: event.outputs, + cache_hit: event.cache_hit, + } + } +} + +impl From for proto::ActionStarted { + fn from(event: events::ActionStarted) -> Self { + proto::ActionStarted { + action_id: event.action_id, + action_name: event.action_name, + target_id: event.target_id, + timestamp_millis: event.timestamp_millis, + } + } +} + +impl From for proto::ActionCompleted { + fn from(event: events::ActionCompleted) -> Self { + proto::ActionCompleted { + action_id: event.action_id, + success: event.success, + timestamp_millis: event.timestamp_millis, + exit_code: event.exit_code, + stdout: event.stdout, + stderr: event.stderr, + } + } +} + +impl From for proto::TargetKind { + fn from(kind: events::TargetKind) -> Self { + match kind { + events::TargetKind::Binary => proto::TargetKind::Binary, + events::TargetKind::Library => proto::TargetKind::Library, + events::TargetKind::Test => proto::TargetKind::Test, + } + } +} + +// ============================================================================ +// Proto -> Rust conversions +// ============================================================================ + +impl TryFrom for events::BuildEvent { + type Error = ConversionError; + + fn try_from(event: proto::BuildEvent) -> Result { + let event = event.event.ok_or(ConversionError::MissingField("event"))?; + + match event { + proto::build_event::Event::BuildStarted(e) => { + Ok(events::BuildEvent::BuildStarted(e.try_into()?)) + } + proto::build_event::Event::BuildFinished(e) => { + Ok(events::BuildEvent::BuildFinished(e.try_into()?)) + } + proto::build_event::Event::TargetStarted(e) => { + Ok(events::BuildEvent::TargetStarted(e.try_into()?)) + } + proto::build_event::Event::TargetCompleted(e) => { + Ok(events::BuildEvent::TargetCompleted(e.try_into()?)) + } + proto::build_event::Event::ActionStarted(e) => { + Ok(events::BuildEvent::ActionStarted(e.try_into()?)) + } + proto::build_event::Event::ActionCompleted(e) => { + Ok(events::BuildEvent::ActionCompleted(e.try_into()?)) + } + } + } +} + +impl TryFrom for events::BuildStarted { + type Error = ConversionError; + + fn try_from(event: proto::BuildStarted) -> Result { + Ok(events::BuildStarted { + invocation_id: event.invocation_id, + command_line: event.command_line, + timestamp_millis: event.timestamp_millis, + working_directory: event.working_directory, + }) + } +} + +impl TryFrom for events::BuildFinished { + type Error = ConversionError; + + fn try_from(event: proto::BuildFinished) -> Result { + Ok(events::BuildFinished { + invocation_id: event.invocation_id, + success: event.success, + timestamp_millis: event.timestamp_millis, + error_message: event.error_message, + }) + } +} + +impl TryFrom for events::TargetStarted { + type Error = ConversionError; + + fn try_from(event: proto::TargetStarted) -> Result { + let target_kind = proto::TargetKind::try_from(event.target_kind) + .map_err(|_| ConversionError::InvalidEnum("target_kind", event.target_kind))?; + + Ok(events::TargetStarted { + target_id: event.target_id, + label: event.label, + target_kind: target_kind.try_into()?, + timestamp_millis: event.timestamp_millis, + }) + } +} + +impl TryFrom for events::TargetCompleted { + type Error = ConversionError; + + fn try_from(event: proto::TargetCompleted) -> Result { + Ok(events::TargetCompleted { + target_id: event.target_id, + label: event.label, + success: event.success, + timestamp_millis: event.timestamp_millis, + error_message: event.error_message, + outputs: event.outputs, + cache_hit: event.cache_hit, + }) + } +} + +impl TryFrom for events::ActionStarted { + type Error = ConversionError; + + fn try_from(event: proto::ActionStarted) -> Result { + Ok(events::ActionStarted { + action_id: event.action_id, + action_name: event.action_name, + target_id: event.target_id, + timestamp_millis: event.timestamp_millis, + }) + } +} + +impl TryFrom for events::ActionCompleted { + type Error = ConversionError; + + fn try_from(event: proto::ActionCompleted) -> Result { + Ok(events::ActionCompleted { + action_id: event.action_id, + success: event.success, + timestamp_millis: event.timestamp_millis, + exit_code: event.exit_code, + stdout: event.stdout, + stderr: event.stderr, + }) + } +} + +impl TryFrom for events::TargetKind { + type Error = ConversionError; + + fn try_from(kind: proto::TargetKind) -> Result { + match kind { + proto::TargetKind::Binary => Ok(events::TargetKind::Binary), + proto::TargetKind::Library => Ok(events::TargetKind::Library), + proto::TargetKind::Test => Ok(events::TargetKind::Test), + proto::TargetKind::Unspecified => Err(ConversionError::InvalidEnum("target_kind", 0)), + } + } +} + +// ============================================================================ +// Error types +// ============================================================================ + +/// Error that can occur during type conversion +#[derive(Debug, thiserror::Error)] +pub enum ConversionError { + #[error("Missing required field: {0}")] + MissingField(&'static str), + + #[error("Invalid enum value for {0}: {1}")] + InvalidEnum(&'static str, i32), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_started_round_trip() { + let rust_event = events::BuildStarted { + invocation_id: "test-123".to_string(), + command_line: vec!["cargo".to_string(), "build".to_string()], + timestamp_millis: 1234567890, + working_directory: "/tmp".to_string(), + }; + + let proto_event: proto::BuildStarted = rust_event.clone().into(); + let converted_back: events::BuildStarted = proto_event.try_into().unwrap(); + + assert_eq!(rust_event, converted_back); + } + + #[test] + fn test_build_event_round_trip() { + let rust_event = events::BuildEvent::BuildStarted(events::BuildStarted { + invocation_id: "test-456".to_string(), + command_line: vec!["cargo".to_string(), "test".to_string()], + timestamp_millis: 9876543210, + working_directory: "/home/user".to_string(), + }); + + let proto_event: proto::BuildEvent = rust_event.clone().into(); + let converted_back: events::BuildEvent = proto_event.try_into().unwrap(); + + assert_eq!(rust_event, converted_back); + } + + #[test] + fn test_target_kind_conversion() { + let rust_kind = events::TargetKind::Binary; + let proto_kind: proto::TargetKind = rust_kind.clone().into(); + let converted_back: events::TargetKind = proto_kind.try_into().unwrap(); + + assert_eq!(rust_kind, converted_back); + } + + #[test] + fn test_optional_fields() { + let rust_event = events::BuildFinished { + invocation_id: "test-789".to_string(), + success: false, + timestamp_millis: 1111111111, + error_message: Some("build failed".to_string()), + }; + + let proto_event: proto::BuildFinished = rust_event.clone().into(); + let converted_back: events::BuildFinished = proto_event.try_into().unwrap(); + + assert_eq!(rust_event, converted_back); + + // Test with None + let rust_event_none = events::BuildFinished { + invocation_id: "test-000".to_string(), + success: true, + timestamp_millis: 2222222222, + error_message: None, + }; + + let proto_event_none: proto::BuildFinished = rust_event_none.clone().into(); + let converted_back_none: events::BuildFinished = proto_event_none.try_into().unwrap(); + + assert_eq!(rust_event_none, converted_back_none); + } +} diff --git a/crates/build-events-proto/src/lib.rs b/crates/build-events-proto/src/lib.rs new file mode 100644 index 0000000000..35ac324fa2 --- /dev/null +++ b/crates/build-events-proto/src/lib.rs @@ -0,0 +1,136 @@ +//! Build Events Protobuf Layer +//! +//! This crate provides protobuf definitions and gRPC streaming support for +//! the `build-events` crate. It enables remote observability of builds via +//! gRPC streaming. +//! +//! # Overview +//! +//! This crate extends the pure Rust `build-events` crate with: +//! - **Protobuf message definitions** for all build event types +//! - **Type conversions** between Rust and protobuf types +//! - **GrpcStreamSubscriber** for converting events to proto and streaming +//! - **BuildEventService** gRPC service implementation +//! +//! # Architecture +//! +//! ```text +//! Build Events (Rust) → Conversion → Proto Events → gRPC Stream → Clients +//! ``` +//! +//! # Example: gRPC Streaming +//! +//! ## Server Side +//! +//! ```no_run +//! use build_events::{BuildEventBus, BuildEventDispatcher}; +//! use build_events_proto::{ +//! GrpcStreamSubscriber, +//! BuildEventServiceImpl, +//! proto::build_event_service_server::BuildEventServiceServer, +//! }; +//! use tonic::transport::Server; +//! +//! #[tokio::main] +//! async fn main() -> Result<(), Box> { +//! // Create event bus +//! let event_bus = BuildEventBus::new(10000); +//! +//! // Create gRPC service +//! let service = BuildEventServiceImpl::new(1000); +//! let sender = service.sender(); +//! +//! // Setup dispatcher with a subscriber that feeds the gRPC service +//! let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); +//! let subscriber = GrpcStreamSubscriber::new(100); +//! dispatcher.add_subscriber(Box::new(subscriber)); +//! +//! // Start dispatcher +//! tokio::spawn(async move { +//! dispatcher.run().await; +//! }); +//! +//! // Start gRPC server +//! Server::builder() +//! .add_service(BuildEventServiceServer::new(service)) +//! .serve("[::1]:50051".parse()?) +//! .await?; +//! +//! Ok(()) +//! } +//! ``` +//! +//! ## Client Side +//! +//! ```no_run +//! use build_events_proto::proto::{ +//! build_event_service_client::BuildEventServiceClient, +//! StreamEventsRequest, +//! }; +//! use tokio_stream::StreamExt; +//! +//! #[tokio::main] +//! async fn main() -> Result<(), Box> { +//! let mut client = BuildEventServiceClient::connect("http://[::1]:50051").await?; +//! +//! let request = StreamEventsRequest { +//! invocation_id: None, +//! start_timestamp_millis: None, +//! }; +//! +//! let mut stream = client.stream_events(request).await?.into_inner(); +//! +//! while let Some(event) = stream.next().await { +//! let event = event?; +//! println!("Received event: {:?}", event); +//! } +//! +//! Ok(()) +//! } +//! ``` +//! +//! # Type Conversions +//! +//! Convert between Rust and protobuf types: +//! +//! ``` +//! use build_events::events::{BuildStarted, current_millis}; +//! use build_events::BuildEvent; +//! use build_events_proto::proto; +//! +//! // Rust -> Proto +//! let rust_event = BuildEvent::BuildStarted(BuildStarted { +//! invocation_id: "test-123".to_string(), +//! command_line: vec!["cargo".to_string(), "build".to_string()], +//! timestamp_millis: current_millis(), +//! working_directory: "/tmp".to_string(), +//! }); +//! +//! let proto_event: proto::BuildEvent = rust_event.clone().into(); +//! +//! // Proto -> Rust +//! let converted_back: BuildEvent = proto_event.try_into().unwrap(); +//! assert_eq!(rust_event, converted_back); +//! ``` + +// Include generated protobuf code +pub mod proto { + tonic::include_proto!("build_events.v1"); +} + +pub mod convert; +pub mod service; +pub mod subscriber; + +#[cfg(feature = "spongebob-subscriber")] +pub mod spongebob_convert; +#[cfg(feature = "spongebob-subscriber")] +pub mod spongebob_subscriber_v2; + +// Re-export main types for convenience +pub use convert::ConversionError; +pub use service::BuildEventServiceImpl; +pub use subscriber::GrpcStreamSubscriber; + +#[cfg(feature = "spongebob-subscriber")] +pub use spongebob_subscriber_v2::SpongeBobSubscriberV2; diff --git a/crates/build-events-proto/src/service.rs b/crates/build-events-proto/src/service.rs new file mode 100644 index 0000000000..e9791e6f75 --- /dev/null +++ b/crates/build-events-proto/src/service.rs @@ -0,0 +1,213 @@ +//! gRPC service implementation for build events +//! +//! Provides a [`BuildEventServiceImpl`] that implements the gRPC service +//! defined in the protobuf schema. + +use tokio::sync::broadcast; +use tokio_stream::wrappers::BroadcastStream; +use tokio_stream::{Stream, StreamExt}; +use tonic::{Request, Response, Status}; +use tracing::{info, warn}; + +use crate::proto; +use crate::proto::build_event_service_server::BuildEventService; + +/// Implementation of the BuildEventService gRPC service +/// +/// This service provides server-side streaming of build events. It uses +/// a broadcast channel to allow multiple clients to subscribe to the same +/// event stream concurrently. +/// +/// # Example +/// ```no_run +/// use build_events_proto::{BuildEventServiceImpl, proto::build_event_service_server::BuildEventServiceServer}; +/// use tonic::transport::Server; +/// +/// #[tokio::main] +/// async fn main() -> Result<(), Box> { +/// let service = BuildEventServiceImpl::new(1000); +/// +/// Server::builder() +/// .add_service(BuildEventServiceServer::new(service)) +/// .serve("[::1]:50051".parse()?) +/// .await?; +/// +/// Ok(()) +/// } +/// ``` +pub struct BuildEventServiceImpl { + event_sender: broadcast::Sender, +} + +impl BuildEventServiceImpl { + /// Create a new build event service with the specified channel capacity + /// + /// # Arguments + /// * `capacity` - Broadcast channel capacity + /// + /// # Example + /// ``` + /// use build_events_proto::BuildEventServiceImpl; + /// + /// let service = BuildEventServiceImpl::new(1000); + /// ``` + pub fn new(capacity: usize) -> Self { + let (sender, _) = broadcast::channel(capacity); + Self { + event_sender: sender, + } + } + + /// Publish an event to all subscribers + /// + /// This method allows the build system to publish events that will + /// be streamed to all connected gRPC clients. + /// + /// # Arguments + /// * `event` - The protobuf event to publish + /// + /// # Example + /// ``` + /// use build_events_proto::{BuildEventServiceImpl, proto}; + /// + /// let service = BuildEventServiceImpl::new(1000); + /// let event = proto::BuildEvent { event: None }; + /// service.publish(event); + /// ``` + pub fn publish(&self, event: proto::BuildEvent) { + // Broadcast to all subscribers + // If there are no subscribers, this is a no-op + let subscriber_count = self.event_sender.receiver_count(); + if subscriber_count == 0 { + return; + } + + if let Err(e) = self.event_sender.send(event) { + warn!("Failed to broadcast event: no receivers ({})", e); + } + } + + /// Get a sender for publishing events + /// + /// This is useful for integrating with the build system's event bus. + pub fn sender(&self) -> broadcast::Sender { + self.event_sender.clone() + } + + /// Get the current number of active subscribers + pub fn subscriber_count(&self) -> usize { + self.event_sender.receiver_count() + } +} + +#[tonic::async_trait] +impl BuildEventService for BuildEventServiceImpl { + type StreamEventsStream = + std::pin::Pin> + Send + 'static>>; + + async fn stream_events( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + info!( + "New event stream subscription: invocation_id={:?}, start_timestamp={:?}", + req.invocation_id, req.start_timestamp_millis + ); + + // Subscribe to the broadcast channel + let receiver = self.event_sender.subscribe(); + let stream = BroadcastStream::new(receiver); + + // Apply filters if specified + let invocation_filter = req.invocation_id.clone(); + let timestamp_filter = req.start_timestamp_millis; + + // Wrap the stream with filtering logic and error mapping + let filtered_stream = stream.filter_map(move |result| { + match result { + Ok(event) => { + // Apply invocation_id filter if specified + if let Some(ref filter_id) = invocation_filter { + // Check if event has matching invocation_id + let matches = match &event.event { + Some(proto::build_event::Event::BuildStarted(e)) => { + &e.invocation_id == filter_id + } + Some(proto::build_event::Event::BuildFinished(e)) => { + &e.invocation_id == filter_id + } + _ => true, // Other events pass through + }; + + if !matches { + return None; + } + } + + // Apply timestamp filter if specified + if let Some(start_ts) = timestamp_filter { + let event_ts = match &event.event { + Some(proto::build_event::Event::BuildStarted(e)) => e.timestamp_millis, + Some(proto::build_event::Event::BuildFinished(e)) => e.timestamp_millis, + Some(proto::build_event::Event::TargetStarted(e)) => e.timestamp_millis, + Some(proto::build_event::Event::TargetCompleted(e)) => { + e.timestamp_millis + } + Some(proto::build_event::Event::ActionStarted(e)) => e.timestamp_millis, + Some(proto::build_event::Event::ActionCompleted(e)) => { + e.timestamp_millis + } + None => return None, + }; + + if event_ts < start_ts { + return None; + } + } + + Some(Ok(event)) + } + Err(e) => { + warn!("Broadcast stream error: {}", e); + // Convert to gRPC status + Some(Err(Status::internal(format!("Stream error: {}", e)))) + } + } + }); + + Ok(Response::new(Box::pin(filtered_stream))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_service_creation() { + let service = BuildEventServiceImpl::new(100); + assert_eq!(service.subscriber_count(), 0); + } + + #[test] + fn test_publish_no_subscribers() { + let service = BuildEventServiceImpl::new(100); + let event = proto::BuildEvent { event: None }; + // Should not panic + service.publish(event); + } + + #[tokio::test] + async fn test_subscriber_count() { + let service = BuildEventServiceImpl::new(100); + assert_eq!(service.subscriber_count(), 0); + + let _rx1 = service.event_sender.subscribe(); + assert_eq!(service.subscriber_count(), 1); + + let _rx2 = service.event_sender.subscribe(); + assert_eq!(service.subscriber_count(), 2); + } +} diff --git a/crates/build-events-proto/src/spongebob_convert.rs b/crates/build-events-proto/src/spongebob_convert.rs new file mode 100644 index 0000000000..2918c8dc5a --- /dev/null +++ b/crates/build-events-proto/src/spongebob_convert.rs @@ -0,0 +1,165 @@ +//! Conversion between build-events Rust types and spongebob proto types +//! +//! This module is only available when the `spongebob-subscriber` feature is enabled. + +use build_events::events::{ + ActionCompleted, ActionStarted, BuildEvent, BuildFinished, BuildStarted, TargetCompleted, + TargetKind, TargetStarted, +}; + +// Re-export the proto types from the spongebob crate +pub use minimal_spongebob_community_neoeinstein_prost::spongebob::v1 as pb; + +/// Convert Rust BuildEvent to spongebob proto BuildEvent +/// Note: This function requires an invocation_id to be passed separately since the +/// Rust BuildEvent enum doesn't carry it at the wrapper level +pub fn to_proto_build_event(event: &BuildEvent, invocation_id: &str) -> pb::BuildEvent { + match event { + BuildEvent::BuildStarted(e) => pb::BuildEvent { + invocation_id: invocation_id.to_string(), + event: Some(pb::build_event::Event::BuildStarted( + to_proto_build_started(e), + )), + }, + BuildEvent::BuildFinished(e) => pb::BuildEvent { + invocation_id: invocation_id.to_string(), + event: Some(pb::build_event::Event::BuildFinished( + to_proto_build_finished(e), + )), + }, + BuildEvent::TargetStarted(e) => pb::BuildEvent { + invocation_id: invocation_id.to_string(), + event: Some(pb::build_event::Event::TargetStarted( + to_proto_target_started(e), + )), + }, + BuildEvent::TargetCompleted(e) => pb::BuildEvent { + invocation_id: invocation_id.to_string(), + event: Some(pb::build_event::Event::TargetCompleted( + to_proto_target_completed(e), + )), + }, + BuildEvent::ActionStarted(e) => pb::BuildEvent { + invocation_id: invocation_id.to_string(), + event: Some(pb::build_event::Event::ActionStarted( + to_proto_action_started(e), + )), + }, + BuildEvent::ActionCompleted(e) => pb::BuildEvent { + invocation_id: invocation_id.to_string(), + event: Some(pb::build_event::Event::ActionCompleted( + to_proto_action_completed(e), + )), + }, + } +} + +fn to_proto_build_started(e: &BuildStarted) -> pb::BuildStarted { + pb::BuildStarted { + invocation_id: e.invocation_id.clone(), + command_line: e.command_line.clone(), + timestamp_millis: e.timestamp_millis, + working_directory: e.working_directory.clone(), + } +} + +fn to_proto_build_finished(e: &BuildFinished) -> pb::BuildFinished { + pb::BuildFinished { + invocation_id: e.invocation_id.clone(), + success: e.success, + timestamp_millis: e.timestamp_millis, + error_message: e.error_message.clone(), + } +} + +fn to_proto_target_started(e: &TargetStarted) -> pb::TargetStarted { + pb::TargetStarted { + target_id: e.target_id.clone(), + label: e.label.clone(), + target_kind: to_proto_target_kind(&e.target_kind) as i32, + timestamp_millis: e.timestamp_millis, + } +} + +fn to_proto_target_completed(e: &TargetCompleted) -> pb::TargetCompleted { + pb::TargetCompleted { + target_id: e.target_id.clone(), + label: e.label.clone(), + success: e.success, + timestamp_millis: e.timestamp_millis, + error_message: e.error_message.clone(), + outputs: e.outputs.clone(), + cache_hit: e.cache_hit, + } +} + +fn to_proto_action_started(e: &ActionStarted) -> pb::ActionStarted { + pb::ActionStarted { + action_id: e.action_id.clone(), + action_name: e.action_name.clone(), + target_id: e.target_id.clone(), + timestamp_millis: e.timestamp_millis, + } +} + +fn to_proto_action_completed(e: &ActionCompleted) -> pb::ActionCompleted { + pb::ActionCompleted { + action_id: e.action_id.clone(), + success: e.success, + timestamp_millis: e.timestamp_millis, + exit_code: e.exit_code, + stdout: e.stdout.clone(), + stderr: e.stderr.clone(), + } +} + +fn to_proto_target_kind(kind: &TargetKind) -> pb::TargetKind { + match kind { + TargetKind::Binary => pb::TargetKind::Binary, + TargetKind::Library => pb::TargetKind::Library, + TargetKind::Test => pb::TargetKind::Test, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use build_events::events::current_millis; + + #[test] + fn test_build_started_conversion() { + let rust_event = BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command_line: vec!["cargo".to_string(), "build".to_string()], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + }); + + let proto_event: pb::BuildEvent = to_proto_build_event(&rust_event, "test-123"); + assert_eq!(proto_event.invocation_id, "test-123"); + assert!(proto_event.event.is_some()); + match proto_event.event.unwrap() { + pb::build_event::Event::BuildStarted(e) => { + assert_eq!(e.invocation_id, "test-123"); + assert_eq!(e.command_line.len(), 2); + } + _ => panic!("Wrong event type"), + } + } + + #[test] + fn test_target_kind_conversion() { + assert_eq!( + to_proto_target_kind(&TargetKind::Binary) as i32, + pb::TargetKind::Binary as i32 + ); + assert_eq!( + to_proto_target_kind(&TargetKind::Library) as i32, + pb::TargetKind::Library as i32 + ); + assert_eq!( + to_proto_target_kind(&TargetKind::Test) as i32, + pb::TargetKind::Test as i32 + ); + } +} diff --git a/crates/build-events-proto/src/spongebob_subscriber_v2.rs b/crates/build-events-proto/src/spongebob_subscriber_v2.rs new file mode 100644 index 0000000000..cd1ae1b924 --- /dev/null +++ b/crates/build-events-proto/src/spongebob_subscriber_v2.rs @@ -0,0 +1,112 @@ +//! Subscriber that sends build events to Spongebob service using the workspace spongebob crate + +#[cfg(feature = "spongebob-subscriber")] +use async_trait::async_trait; +#[cfg(feature = "spongebob-subscriber")] +use build_events::{BuildEvent, BuildEventSubscriber, SubscriberError}; +#[cfg(feature = "spongebob-subscriber")] +use tracing::warn; + +#[cfg(feature = "spongebob-subscriber")] +use crate::spongebob_convert::to_proto_build_event; + +/// Subscriber that publishes build events to Spongebob service +/// +/// This subscriber uses the workspace `spongebob` crate to connect to the +/// Spongebob service and publish build events. +/// +/// # Example +/// ```no_run +/// # #[cfg(feature = "spongebob-subscriber")] +/// use build_events_proto::SpongeBobSubscriberV2; +/// +/// #[tokio::main] +/// async fn main() -> Result<(), Box> { +/// // Create a spongebob client and invocation +/// let mut spongebob = spongebob::SpongeBob::new().await?; +/// let invocation = spongebob.create_invocation("my-build").await?; +/// +/// // Create subscriber from the invocation +/// let subscriber = SpongeBobSubscriberV2::from_invocation(invocation); +/// +/// // Use with BuildEventDispatcher +/// // dispatcher.add_subscriber(Box::new(subscriber)); +/// Ok(()) +/// } +/// ``` +#[cfg(feature = "spongebob-subscriber")] +pub struct SpongeBobSubscriberV2 { + invocation: std::sync::Arc>, +} + +#[cfg(feature = "spongebob-subscriber")] +impl SpongeBobSubscriberV2 { + /// Create subscriber from an existing SpongeBobInvocation + /// + /// # Arguments + /// * `invocation` - An existing SpongeBobInvocation to publish events to + /// + /// # Example + /// ```no_run + /// # #[cfg(feature = "spongebob-subscriber")] + /// # use build_events_proto::SpongeBobSubscriberV2; + /// # async fn example() -> Result<(), Box> { + /// let mut spongebob = spongebob::SpongeBob::new().await?; + /// let invocation = spongebob.create_invocation("my-build").await?; + /// let subscriber = SpongeBobSubscriberV2::from_invocation(invocation); + /// # Ok(()) + /// # } + /// ``` + pub fn from_invocation(invocation: spongebob::SpongeBobInvocation) -> Self { + Self { + invocation: std::sync::Arc::new(tokio::sync::Mutex::new(invocation)), + } + } +} + +#[cfg(feature = "spongebob-subscriber")] +#[async_trait] +impl BuildEventSubscriber for SpongeBobSubscriberV2 { + async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { + // Get lock on invocation + let mut invocation = self.invocation.lock().await; + + // Convert Rust event to proto (need invocation_id from the invocation) + let proto_event = to_proto_build_event(event, invocation.invocation_id()); + + // Publish event + match invocation.publish_build_event(proto_event).await { + Ok(()) => { + tracing::debug!("Successfully published build event to Spongebob"); + Ok(()) + } + Err(e) => { + // Log warning but don't fail the build + warn!("Failed to publish build event to Spongebob: {}", e); + // Return error so dispatcher can decide how to handle it + Err(SubscriberError::Custom(format!( + "Failed to publish event: {}", + e + ))) + } + } + } + + fn name(&self) -> &str { + "SpongeBobSubscriberV2" + } + + async fn on_close(&self) -> Result<(), SubscriberError> { + // No cleanup needed + Ok(()) + } +} + +#[cfg(all(test, feature = "spongebob-subscriber"))] +mod tests { + #[test] + fn test_subscriber_creation() { + // This test would require a mock SpongeBobInvocation + // For now, just ensure the module compiles + } +} diff --git a/crates/build-events-proto/src/subscriber.rs b/crates/build-events-proto/src/subscriber.rs new file mode 100644 index 0000000000..2abf8f330c --- /dev/null +++ b/crates/build-events-proto/src/subscriber.rs @@ -0,0 +1,156 @@ +//! gRPC streaming subscriber implementation +//! +//! Provides a [`GrpcStreamSubscriber`] that converts Rust events to protobuf +//! and makes them available as an async stream. + +use async_trait::async_trait; +use build_events::{BuildEvent, BuildEventSubscriber, SubscriberError}; +use tokio::sync::broadcast; +use tokio_stream::Stream; +use tokio_stream::wrappers::BroadcastStream; +use tracing::warn; + +use crate::proto; + +/// Subscriber that converts events to protobuf and provides them as a stream +/// +/// This subscriber implements [`BuildEventSubscriber`] and maintains an internal +/// broadcast channel for streaming events. It converts Rust events to protobuf +/// on reception and broadcasts them to all subscribers. +/// +/// # Example +/// ``` +/// use build_events_proto::GrpcStreamSubscriber; +/// +/// // Create subscriber +/// let subscriber = GrpcStreamSubscriber::new(100); +/// +/// // Can be used with BuildEventDispatcher +/// // dispatcher.add_subscriber(Box::new(subscriber)); +/// ``` +pub struct GrpcStreamSubscriber { + sender: broadcast::Sender, +} + +impl GrpcStreamSubscriber { + /// Create a new gRPC stream subscriber with the specified channel capacity + /// + /// The capacity determines how many events can be buffered before + /// slow subscribers start lagging. + /// + /// # Arguments + /// * `capacity` - Broadcast channel buffer capacity + /// + /// # Example + /// ``` + /// use build_events_proto::GrpcStreamSubscriber; + /// + /// let subscriber = GrpcStreamSubscriber::new(100); + /// ``` + pub fn new(capacity: usize) -> Self { + let (sender, _) = broadcast::channel(capacity); + Self { sender } + } + + /// Subscribe to the event stream + /// + /// Returns a stream that receives all events broadcasted by this subscriber. + /// Multiple subscriptions can be created, and each will receive all events. + /// + /// # Example + /// ``` + /// use build_events_proto::GrpcStreamSubscriber; + /// use tokio_stream::StreamExt; + /// + /// # #[tokio::main] + /// # async fn main() { + /// let subscriber = GrpcStreamSubscriber::new(100); + /// let mut stream = subscriber.subscribe_stream(); + /// + /// // Consume events + /// // while let Some(Ok(event)) = stream.next().await { + /// // // Process event + /// // } + /// # } + /// ``` + pub fn subscribe_stream(&self) -> impl Stream { + use tokio_stream::StreamExt; + BroadcastStream::new(self.sender.subscribe()).filter_map(|result| result.ok()) + } + + /// Get a broadcast sender for this subscriber + /// + /// Useful for advanced use cases where you need direct access to the sender. + pub fn sender(&self) -> broadcast::Sender { + self.sender.clone() + } +} + +#[async_trait] +impl BuildEventSubscriber for GrpcStreamSubscriber { + async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { + // Convert Rust event to proto + let proto_event: proto::BuildEvent = event.clone().into(); + + // Broadcast to all subscribers + // If there are no subscribers, this is a no-op + if self.sender.receiver_count() == 0 { + return Ok(()); + } + + // Send to broadcast channel (will drop if no receivers) + if let Err(e) = self.sender.send(proto_event) { + // If channel has no receivers, this is expected + warn!("GrpcStreamSubscriber: no active receivers ({})", e); + } + + Ok(()) + } + + fn name(&self) -> &str { + "GrpcStreamSubscriber" + } + + async fn on_close(&self) -> Result<(), SubscriberError> { + // Channel will automatically close when sender is dropped + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use build_events::events::{BuildStarted, current_millis}; + use tokio_stream::StreamExt; + + #[tokio::test] + async fn test_subscriber_stream() { + let subscriber = GrpcStreamSubscriber::new(10); + let mut stream = subscriber.subscribe_stream(); + + let event = BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command_line: vec!["cargo".to_string(), "test".to_string()], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + }); + + // Send event + subscriber.on_event(&event).await.unwrap(); + + // Receive from stream + let received = tokio::time::timeout(std::time::Duration::from_millis(100), stream.next()) + .await + .unwrap(); + + assert!(received.is_some()); + let proto_event = received.unwrap(); + assert!(proto_event.event.is_some()); + } + + #[tokio::test] + async fn test_subscriber_name() { + let subscriber = GrpcStreamSubscriber::new(10); + assert_eq!(subscriber.name(), "GrpcStreamSubscriber"); + } +} diff --git a/crates/build-events/Cargo.toml b/crates/build-events/Cargo.toml new file mode 100644 index 0000000000..d74c62de97 --- /dev/null +++ b/crates/build-events/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "build-events" +version = "0.0.1" +edition.workspace = true +rust-version = "1.85" +license = "MIT OR Apache-2.0" +publish.workspace = true + +[dependencies] +tokio.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tracing.workspace = true +async-trait = "0.1" + +[dev-dependencies] +serde_json.workspace = true +tempfile.workspace = true +tracing-subscriber.workspace = true diff --git a/crates/build-events/examples/basic_usage.rs b/crates/build-events/examples/basic_usage.rs new file mode 100644 index 0000000000..3697824d18 --- /dev/null +++ b/crates/build-events/examples/basic_usage.rs @@ -0,0 +1,169 @@ +//! Basic usage example for the build-events crate +//! +//! This example demonstrates: +//! - Creating a BuildEventBus +//! - Setting up multiple subscribers (LoggerSubscriber and JsonFileWriter) +//! - Emitting all event types +//! - Graceful shutdown + +use build_events::{ + BuildEvent, BuildEventBus, BuildEventDispatcher, + events::{ + ActionCompleted, ActionStarted, BuildFinished, BuildStarted, TargetCompleted, TargetKind, + TargetStarted, current_millis, + }, + subscribers::{JsonFileWriter, LoggerSubscriber}, +}; + +#[tokio::main] +async fn main() { + // Initialize tracing for LoggerSubscriber + tracing_subscriber::fmt() + .with_target(false) + .with_thread_ids(false) + .with_file(false) + .with_line_number(false) + .init(); + + println!("=== Build Events System Demo ===\n"); + + // Create event bus with capacity for 10,000 events + let event_bus = BuildEventBus::new(10000); + println!("Created BuildEventBus with capacity 10,000"); + + // Setup subscribers + let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); + + // Add LoggerSubscriber to emit events to tracing + dispatcher.add_subscriber(Box::new(LoggerSubscriber::new())); + + // Add JsonFileWriter to write events to a file + let json_path = "build_events.jsonl"; + dispatcher.add_subscriber(Box::new( + JsonFileWriter::new(json_path).expect("Failed to create JSON file writer"), + )); + + println!("Registered subscribers: LoggerSubscriber, JsonFileWriter"); + println!("Events will be written to: {}\n", json_path); + + // Start event processing in background + let dispatcher_handle = tokio::spawn(async move { + dispatcher.run().await; + }); + + println!("Started event dispatcher\n"); + + // Simulate a build by emitting events + simulate_build(&event_bus).await; + + println!("\nBuild simulation complete"); + + // Shutdown: drop event_bus to close channel, wait for dispatcher to drain + println!("Shutting down event bus..."); + drop(event_bus); + + // Wait for dispatcher to finish processing + dispatcher_handle.await.expect("Dispatcher task failed"); + + println!("Event dispatcher shutdown complete\n"); + println!("=== Demo Complete ==="); + println!("\nYou can view the emitted events in: {}", json_path); +} + +async fn simulate_build(bus: &BuildEventBus) { + println!("--- Simulating Build ---\n"); + + // 1. Build started + println!("1. Emitting BuildStarted event"); + bus.emit(BuildEvent::BuildStarted(BuildStarted { + invocation_id: "demo-invocation-123".to_string(), + command_line: vec![ + "minimal".to_string(), + "build".to_string(), + "//example:target".to_string(), + ], + timestamp_millis: current_millis(), + working_directory: std::env::current_dir() + .unwrap() + .to_string_lossy() + .to_string(), + })); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + // 2. Target started + println!("2. Emitting TargetStarted event"); + bus.emit(BuildEvent::TargetStarted(TargetStarted { + target_id: "target-1".to_string(), + label: "//example:target".to_string(), + target_kind: TargetKind::Binary, + timestamp_millis: current_millis(), + })); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + // 3. Action started - compile + println!("3. Emitting ActionStarted event (compile)"); + bus.emit(BuildEvent::ActionStarted(ActionStarted { + action_id: "action-compile-1".to_string(), + action_name: "RustcCompile".to_string(), + target_id: "target-1".to_string(), + timestamp_millis: current_millis(), + })); + tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; + + // 4. Action completed - compile + println!("4. Emitting ActionCompleted event (compile)"); + bus.emit(BuildEvent::ActionCompleted(ActionCompleted { + action_id: "action-compile-1".to_string(), + success: true, + timestamp_millis: current_millis(), + exit_code: 0, + stdout: Some("Compiling example v0.1.0".to_string()), + stderr: None, + })); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + // 5. Action started - link + println!("5. Emitting ActionStarted event (link)"); + bus.emit(BuildEvent::ActionStarted(ActionStarted { + action_id: "action-link-1".to_string(), + action_name: "RustcLink".to_string(), + target_id: "target-1".to_string(), + timestamp_millis: current_millis(), + })); + tokio::time::sleep(tokio::time::Duration::from_millis(150)).await; + + // 6. Action completed - link + println!("6. Emitting ActionCompleted event (link)"); + bus.emit(BuildEvent::ActionCompleted(ActionCompleted { + action_id: "action-link-1".to_string(), + success: true, + timestamp_millis: current_millis(), + exit_code: 0, + stdout: Some("Finished release [optimized] target(s)".to_string()), + stderr: None, + })); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + // 7. Target completed + println!("7. Emitting TargetCompleted event"); + bus.emit(BuildEvent::TargetCompleted(TargetCompleted { + target_id: "target-1".to_string(), + label: "//example:target".to_string(), + success: true, + timestamp_millis: current_millis(), + error_message: None, + outputs: vec!["target/release/example".to_string()], + cache_hit: false, + })); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + // 8. Build finished + println!("8. Emitting BuildFinished event"); + bus.emit(BuildEvent::BuildFinished(BuildFinished { + invocation_id: "demo-invocation-123".to_string(), + success: true, + timestamp_millis: current_millis(), + error_message: None, + })); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; +} diff --git a/crates/build-events/src/bus.rs b/crates/build-events/src/bus.rs new file mode 100644 index 0000000000..264900cf32 --- /dev/null +++ b/crates/build-events/src/bus.rs @@ -0,0 +1,235 @@ +//! Build event bus implementation +//! +//! The [`BuildEventBus`] is the central event dispatcher that uses a tokio +//! broadcast channel to distribute events to multiple subscribers. + +use crate::events::BuildEvent; +use std::sync::Arc; +use tokio::sync::broadcast; + +/// Central event bus for distributing build events to subscribers +/// +/// The bus uses a tokio broadcast channel internally and can be cheaply cloned +/// to share across multiple components. Events are emitted in a non-blocking +/// manner using try_send. +/// +/// # Example +/// ``` +/// use build_events::{BuildEventBus, BuildEvent, events::*}; +/// +/// #[tokio::main] +/// async fn main() { +/// let bus = BuildEventBus::new(1000); +/// let mut receiver = bus.subscribe(); +/// +/// bus.emit(BuildEvent::BuildStarted(BuildStarted { +/// invocation_id: "test-123".to_string(), +/// command_line: vec!["cargo".to_string(), "build".to_string()], +/// timestamp_millis: current_millis(), +/// working_directory: "/tmp".to_string(), +/// })); +/// +/// let event = receiver.recv().await.unwrap(); +/// match event { +/// BuildEvent::BuildStarted(_) => println!("Build started!"), +/// _ => {} +/// } +/// } +/// ``` +#[derive(Clone, Debug)] +pub struct BuildEventBus { + sender: Arc>, +} + +impl BuildEventBus { + /// Create a new build event bus with the specified channel capacity + /// + /// The capacity determines how many events can be buffered before + /// slow subscribers start lagging and missing events. + /// + /// # Arguments + /// * `capacity` - Maximum number of events to buffer + /// + /// # Example + /// ``` + /// use build_events::BuildEventBus; + /// + /// let bus = BuildEventBus::new(10000); + /// ``` + pub fn new(capacity: usize) -> Self { + let (sender, _) = broadcast::channel(capacity); + Self { + sender: Arc::new(sender), + } + } + + /// Emit an event to all subscribers + /// + /// This is a non-blocking operation. If the channel is full, the event + /// will be dropped and slow subscribers may miss it. This ensures that + /// event emission never blocks the build process. + /// + /// # Arguments + /// * `event` - The event to emit + /// + /// # Example + /// ``` + /// use build_events::{BuildEventBus, BuildEvent, events::*}; + /// + /// let bus = BuildEventBus::new(1000); + /// bus.emit(BuildEvent::BuildStarted(BuildStarted { + /// invocation_id: "test-123".to_string(), + /// command_line: vec!["cargo".to_string(), "build".to_string()], + /// timestamp_millis: current_millis(), + /// working_directory: "/tmp".to_string(), + /// })); + /// ``` + pub fn emit(&self, event: BuildEvent) { + // Use try_send to avoid blocking. If the channel is full, the event + // will be dropped. This is acceptable because we prioritize not + // blocking the build process over guaranteed delivery. + let _ = self.sender.send(event); + } + + /// Subscribe to build events + /// + /// Creates a new receiver that will receive all events emitted after + /// this subscription is created. Events emitted before subscription + /// will not be received. + /// + /// # Example + /// ``` + /// use build_events::BuildEventBus; + /// + /// let bus = BuildEventBus::new(1000); + /// let receiver1 = bus.subscribe(); + /// let receiver2 = bus.subscribe(); + /// // Both receivers will get events emitted from now on + /// ``` + pub fn subscribe(&self) -> broadcast::Receiver { + self.sender.subscribe() + } + + /// Get the current number of active subscribers + /// + /// # Example + /// ``` + /// use build_events::BuildEventBus; + /// + /// let bus = BuildEventBus::new(1000); + /// assert_eq!(bus.receiver_count(), 0); + /// let _receiver = bus.subscribe(); + /// assert_eq!(bus.receiver_count(), 1); + /// ``` + pub fn receiver_count(&self) -> usize { + self.sender.receiver_count() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::{BuildStarted, current_millis}; + + #[tokio::test] + async fn test_bus_emit_and_receive() { + let bus = BuildEventBus::new(100); + let mut receiver = bus.subscribe(); + + let event = BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command_line: vec!["cargo".to_string(), "build".to_string()], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + }); + + bus.emit(event.clone()); + + let received = receiver.recv().await.unwrap(); + assert_eq!(received, event); + } + + #[tokio::test] + async fn test_bus_multiple_subscribers() { + let bus = BuildEventBus::new(100); + let mut receiver1 = bus.subscribe(); + let mut receiver2 = bus.subscribe(); + + let event = BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test-456".to_string(), + command_line: vec!["cargo".to_string(), "test".to_string()], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + }); + + bus.emit(event.clone()); + + let received1 = receiver1.recv().await.unwrap(); + let received2 = receiver2.recv().await.unwrap(); + assert_eq!(received1, event); + assert_eq!(received2, event); + } + + #[tokio::test] + async fn test_bus_clone() { + let bus = BuildEventBus::new(100); + let bus_clone = bus.clone(); + let mut receiver = bus.subscribe(); + + let event = BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test-789".to_string(), + command_line: vec!["cargo".to_string(), "run".to_string()], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + }); + + // Emit from the cloned bus + bus_clone.emit(event.clone()); + + let received = receiver.recv().await.unwrap(); + assert_eq!(received, event); + } + + #[test] + fn test_receiver_count() { + let bus = BuildEventBus::new(100); + assert_eq!(bus.receiver_count(), 0); + + let _receiver1 = bus.subscribe(); + assert_eq!(bus.receiver_count(), 1); + + let _receiver2 = bus.subscribe(); + assert_eq!(bus.receiver_count(), 2); + } + + #[tokio::test] + async fn test_late_subscription() { + let bus = BuildEventBus::new(100); + + let event1 = BuildEvent::BuildStarted(BuildStarted { + invocation_id: "early".to_string(), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + }); + + // Emit before subscription + bus.emit(event1.clone()); + + // Subscribe after event was emitted + let mut receiver = bus.subscribe(); + + let event2 = BuildEvent::BuildStarted(BuildStarted { + invocation_id: "late".to_string(), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + }); + + bus.emit(event2.clone()); + + // Should only receive event2, not event1 + let received = receiver.recv().await.unwrap(); + assert_eq!(received, event2); + } +} diff --git a/crates/build-events/src/dispatcher.rs b/crates/build-events/src/dispatcher.rs new file mode 100644 index 0000000000..a81302662b --- /dev/null +++ b/crates/build-events/src/dispatcher.rs @@ -0,0 +1,307 @@ +//! Build event dispatcher implementation +//! +//! The [`BuildEventDispatcher`] manages multiple subscribers and distributes +//! events from a broadcast channel to all registered subscribers. + +use crate::events::BuildEvent; +use crate::subscriber::BuildEventSubscriber; +use tokio::sync::broadcast; +use tracing::{info, warn}; + +/// Dispatcher that distributes events to multiple subscribers +/// +/// The dispatcher receives events from a broadcast channel and forwards +/// them to all registered subscribers. It handles subscriber errors +/// gracefully by logging them without stopping other subscribers. +/// +/// # Example +/// ``` +/// use build_events::{BuildEventBus, BuildEventDispatcher, subscribers::LoggerSubscriber}; +/// +/// #[tokio::main] +/// async fn main() { +/// let bus = BuildEventBus::new(1000); +/// let mut dispatcher = BuildEventDispatcher::new(bus.subscribe()); +/// +/// dispatcher.add_subscriber(Box::new(LoggerSubscriber::new())); +/// +/// tokio::spawn(async move { +/// dispatcher.run().await; +/// }); +/// +/// // Use the bus to emit events... +/// } +/// ``` +pub struct BuildEventDispatcher { + receiver: broadcast::Receiver, + subscribers: Vec>, +} + +impl BuildEventDispatcher { + /// Create a new dispatcher with a broadcast receiver + /// + /// # Arguments + /// * `receiver` - Broadcast receiver from the event bus + /// + /// # Example + /// ``` + /// use build_events::{BuildEventBus, BuildEventDispatcher}; + /// + /// let bus = BuildEventBus::new(1000); + /// let dispatcher = BuildEventDispatcher::new(bus.subscribe()); + /// ``` + pub fn new(receiver: broadcast::Receiver) -> Self { + Self { + receiver, + subscribers: Vec::new(), + } + } + + /// Add a subscriber to receive events + /// + /// # Arguments + /// * `subscriber` - Boxed subscriber implementation + /// + /// # Example + /// ``` + /// use build_events::{BuildEventBus, BuildEventDispatcher, subscribers::LoggerSubscriber}; + /// + /// let bus = BuildEventBus::new(1000); + /// let mut dispatcher = BuildEventDispatcher::new(bus.subscribe()); + /// dispatcher.add_subscriber(Box::new(LoggerSubscriber::new())); + /// ``` + pub fn add_subscriber(&mut self, subscriber: Box) { + info!("Registered subscriber: {}", subscriber.name()); + self.subscribers.push(subscriber); + } + + /// Run the dispatcher, processing events until the channel closes + /// + /// This method will block until the broadcast channel is closed (which + /// happens when all senders are dropped). It distributes each event + /// to all registered subscribers. + /// + /// Subscriber errors are logged via `tracing::warn!` but don't stop + /// event processing or affect other subscribers. + /// + /// # Example + /// ```no_run + /// use build_events::{BuildEventBus, BuildEventDispatcher, subscribers::LoggerSubscriber}; + /// + /// #[tokio::main] + /// async fn main() { + /// let bus = BuildEventBus::new(1000); + /// let mut dispatcher = BuildEventDispatcher::new(bus.subscribe()); + /// dispatcher.add_subscriber(Box::new(LoggerSubscriber::new())); + /// + /// // Run dispatcher until bus is closed + /// dispatcher.run().await; + /// } + /// ``` + pub async fn run(mut self) { + info!( + "BuildEventDispatcher started with {} subscribers", + self.subscribers.len() + ); + + loop { + match self.receiver.recv().await { + Ok(event) => { + // Distribute event to all subscribers + for subscriber in &self.subscribers { + if let Err(e) = subscriber.on_event(&event).await { + warn!( + "Subscriber '{}' error processing event: {}", + subscriber.name(), + e + ); + } + } + } + Err(broadcast::error::RecvError::Closed) => { + // Channel closed, shutdown gracefully + info!("Event channel closed, shutting down dispatcher"); + break; + } + Err(broadcast::error::RecvError::Lagged(skipped)) => { + // This receiver is too slow and missed some events + warn!("Dispatcher lagged and skipped {} events", skipped); + } + } + } + + // Call on_close for all subscribers + for subscriber in &self.subscribers { + if let Err(e) = subscriber.on_close().await { + warn!( + "Subscriber '{}' error during close: {}", + subscriber.name(), + e + ); + } + } + + info!("BuildEventDispatcher shutdown complete"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::{BuildStarted, current_millis}; + use crate::subscriber::SubscriberError; + use async_trait::async_trait; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct CountingSubscriber { + name: String, + count: Arc, + } + + #[async_trait] + impl BuildEventSubscriber for CountingSubscriber { + async fn on_event(&self, _event: &BuildEvent) -> Result<(), SubscriberError> { + self.count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn name(&self) -> &str { + &self.name + } + } + + #[tokio::test] + async fn test_dispatcher_single_subscriber() { + let (sender, receiver) = broadcast::channel(100); + let count = Arc::new(AtomicUsize::new(0)); + + let mut dispatcher = BuildEventDispatcher::new(receiver); + dispatcher.add_subscriber(Box::new(CountingSubscriber { + name: "counter".to_string(), + count: count.clone(), + })); + + // Spawn dispatcher + let handle = tokio::spawn(async move { + dispatcher.run().await; + }); + + // Send some events + for i in 0..5 { + sender + .send(BuildEvent::BuildStarted(BuildStarted { + invocation_id: format!("test-{}", i), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })) + .unwrap(); + } + + // Wait a bit for events to process + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + // Close channel + drop(sender); + + // Wait for dispatcher to finish + handle.await.unwrap(); + + assert_eq!(count.load(Ordering::SeqCst), 5); + } + + #[tokio::test] + async fn test_dispatcher_multiple_subscribers() { + let (sender, receiver) = broadcast::channel(100); + let count1 = Arc::new(AtomicUsize::new(0)); + let count2 = Arc::new(AtomicUsize::new(0)); + + let mut dispatcher = BuildEventDispatcher::new(receiver); + dispatcher.add_subscriber(Box::new(CountingSubscriber { + name: "counter1".to_string(), + count: count1.clone(), + })); + dispatcher.add_subscriber(Box::new(CountingSubscriber { + name: "counter2".to_string(), + count: count2.clone(), + })); + + let handle = tokio::spawn(async move { + dispatcher.run().await; + }); + + // Send events + for i in 0..3 { + sender + .send(BuildEvent::BuildStarted(BuildStarted { + invocation_id: format!("test-{}", i), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })) + .unwrap(); + } + + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + drop(sender); + handle.await.unwrap(); + + // Both subscribers should have received all events + assert_eq!(count1.load(Ordering::SeqCst), 3); + assert_eq!(count2.load(Ordering::SeqCst), 3); + } + + struct ErrorSubscriber { + name: String, + } + + #[async_trait] + impl BuildEventSubscriber for ErrorSubscriber { + async fn on_event(&self, _event: &BuildEvent) -> Result<(), SubscriberError> { + Err(SubscriberError::Custom("intentional error".to_string())) + } + + fn name(&self) -> &str { + &self.name + } + } + + #[tokio::test] + async fn test_dispatcher_subscriber_error() { + let (sender, receiver) = broadcast::channel(100); + let count = Arc::new(AtomicUsize::new(0)); + + let mut dispatcher = BuildEventDispatcher::new(receiver); + // Add an error subscriber and a counting subscriber + dispatcher.add_subscriber(Box::new(ErrorSubscriber { + name: "error".to_string(), + })); + dispatcher.add_subscriber(Box::new(CountingSubscriber { + name: "counter".to_string(), + count: count.clone(), + })); + + let handle = tokio::spawn(async move { + dispatcher.run().await; + }); + + // Send event + sender + .send(BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test".to_string(), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })) + .unwrap(); + + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + drop(sender); + handle.await.unwrap(); + + // Counting subscriber should still have received the event + // even though error subscriber failed + assert_eq!(count.load(Ordering::SeqCst), 1); + } +} diff --git a/crates/build-events/src/events.rs b/crates/build-events/src/events.rs new file mode 100644 index 0000000000..f601a035c5 --- /dev/null +++ b/crates/build-events/src/events.rs @@ -0,0 +1,175 @@ +//! Build event type definitions +//! +//! This module defines the core event types emitted during a build. +//! Events are strongly-typed Rust enums and structs with serde support +//! for easy serialization to JSON, protobuf, or other formats. + +use serde::{Deserialize, Serialize}; + +/// Main build event enum that wraps all possible event types +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum BuildEvent { + /// Build process started + BuildStarted(BuildStarted), + /// Build process finished + BuildFinished(BuildFinished), + /// Target started building + TargetStarted(TargetStarted), + /// Target completed building + TargetCompleted(TargetCompleted), + /// Action started executing + ActionStarted(ActionStarted), + /// Action completed executing + ActionCompleted(ActionCompleted), +} + +/// Event emitted when a build starts +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct BuildStarted { + /// Unique identifier for this build invocation + pub invocation_id: String, + /// Command line arguments used to start the build + pub command_line: Vec, + /// Unix timestamp in milliseconds + pub timestamp_millis: i64, + /// Working directory where build was started + pub working_directory: String, +} + +/// Event emitted when a build finishes +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct BuildFinished { + /// Unique identifier for this build invocation + pub invocation_id: String, + /// Whether the build succeeded + pub success: bool, + /// Unix timestamp in milliseconds + pub timestamp_millis: i64, + /// Optional error message if build failed + pub error_message: Option, +} + +/// Event emitted when a target starts building +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct TargetStarted { + /// Unique identifier for this target + pub target_id: String, + /// Target label (e.g., "//foo:bar") + pub label: String, + /// Type of target being built + pub target_kind: TargetKind, + /// Unix timestamp in milliseconds + pub timestamp_millis: i64, +} + +/// Event emitted when a target completes +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct TargetCompleted { + /// Unique identifier for this target (must match TargetStarted.target_id) + pub target_id: String, + /// Target label (e.g., "//foo:bar") + pub label: String, + /// Whether the target built successfully + pub success: bool, + /// Unix timestamp in milliseconds + pub timestamp_millis: i64, + /// Optional error message if target failed + pub error_message: Option, + /// Paths to generated output files + pub outputs: Vec, + /// Whether this target was served from cache + pub cache_hit: bool, +} + +/// Event emitted when an action starts executing +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ActionStarted { + /// Unique identifier for this action + pub action_id: String, + /// Human-readable name for the action (e.g., "CppCompile", "Link") + pub action_name: String, + /// Target this action belongs to (if any) + pub target_id: String, + /// Unix timestamp in milliseconds + pub timestamp_millis: i64, +} + +/// Event emitted when an action completes +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct ActionCompleted { + /// Unique identifier for this action + pub action_id: String, + /// Whether the action succeeded + pub success: bool, + /// Unix timestamp in milliseconds + pub timestamp_millis: i64, + /// Exit code of the action + pub exit_code: i32, + /// Optional stdout from the action + pub stdout: Option, + /// Optional stderr from the action + pub stderr: Option, +} + +/// Type of build target +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum TargetKind { + /// Binary executable target + Binary, + /// Library target + Library, + /// Test target + Test, +} + +/// Helper function to get current timestamp in milliseconds +/// +/// # Example +/// ``` +/// use build_events::events::current_millis; +/// +/// let timestamp = current_millis(); +/// assert!(timestamp > 0); +/// ``` +pub fn current_millis() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("System time before UNIX epoch") + .as_millis() as i64 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_current_millis() { + let t1 = current_millis(); + std::thread::sleep(std::time::Duration::from_millis(10)); + let t2 = current_millis(); + assert!(t2 > t1); + } + + #[test] + fn test_event_serialization() { + let event = BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command_line: vec!["cargo".to_string(), "build".to_string()], + timestamp_millis: 1234567890, + working_directory: "/tmp".to_string(), + }); + + let json = serde_json::to_string(&event).unwrap(); + let deserialized: BuildEvent = serde_json::from_str(&json).unwrap(); + assert_eq!(event, deserialized); + } + + #[test] + fn test_target_kind_serialization() { + let kind = TargetKind::Binary; + let json = serde_json::to_string(&kind).unwrap(); + assert_eq!(json, "\"binary\""); + } +} diff --git a/crates/build-events/src/lib.rs b/crates/build-events/src/lib.rs new file mode 100644 index 0000000000..b40b0a8def --- /dev/null +++ b/crates/build-events/src/lib.rs @@ -0,0 +1,130 @@ +//! Build Events System +//! +//! A dedicated build event protocol (BEP) system that enables remote observability +//! of build execution. This system is separate from diagnostic tracing and emits +//! structured, strongly-typed events that can be consumed by local and remote services. +//! +//! # Overview +//! +//! The build-events crate provides: +//! - **Strongly-typed event definitions** as pure Rust enums and structs +//! - **BuildEventBus** for publishing events using tokio broadcast channels +//! - **BuildEventSubscriber** trait for consuming events +//! - **BuildEventDispatcher** for managing multiple subscribers +//! - **Built-in subscribers** for logging and JSON file output +//! +//! # Architecture +//! +//! ```text +//! Build Components → BuildEventBus → BuildEventDispatcher → Subscribers +//! ├─ LoggerSubscriber +//! ├─ JsonFileWriter +//! └─ Custom Subscribers +//! ``` +//! +//! # Example +//! +//! ```no_run +//! use build_events::{ +//! BuildEventBus, BuildEventDispatcher, BuildEvent, +//! events::{BuildStarted, current_millis}, +//! subscribers::{LoggerSubscriber, JsonFileWriter}, +//! }; +//! +//! #[tokio::main] +//! async fn main() { +//! // Initialize tracing for LoggerSubscriber +//! tracing_subscriber::fmt::init(); +//! +//! // Create event bus +//! let event_bus = BuildEventBus::new(10000); +//! +//! // Setup subscribers +//! let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); +//! dispatcher.add_subscriber(Box::new(LoggerSubscriber::new())); +//! dispatcher.add_subscriber(Box::new( +//! JsonFileWriter::new("build_events.jsonl").unwrap() +//! )); +//! +//! // Start event processing +//! let dispatcher_handle = tokio::spawn(async move { +//! dispatcher.run().await +//! }); +//! +//! // Emit events from build +//! event_bus.emit(BuildEvent::BuildStarted(BuildStarted { +//! invocation_id: "build-123".to_string(), +//! command_line: std::env::args().collect(), +//! timestamp_millis: current_millis(), +//! working_directory: std::env::current_dir() +//! .unwrap() +//! .to_string_lossy() +//! .to_string(), +//! })); +//! +//! // ... perform build ... +//! +//! // Shutdown: drop event_bus, wait for dispatcher to drain +//! drop(event_bus); +//! let _ = dispatcher_handle.await; +//! } +//! ``` +//! +//! # Event Types +//! +//! The system includes 6 core event types: +//! - [`BuildStarted`](events::BuildStarted) / [`BuildFinished`](events::BuildFinished) +//! - [`TargetStarted`](events::TargetStarted) / [`TargetCompleted`](events::TargetCompleted) +//! - [`ActionStarted`](events::ActionStarted) / [`ActionCompleted`](events::ActionCompleted) +//! +//! All events are strongly-typed Rust structs with `Clone`, `Debug`, `Serialize`, +//! and `Deserialize` implementations. +//! +//! # Custom Subscribers +//! +//! You can implement custom subscribers by implementing the +//! [`BuildEventSubscriber`] trait: +//! +//! ``` +//! use build_events::{BuildEvent, BuildEventSubscriber, SubscriberError}; +//! use async_trait::async_trait; +//! +//! struct MySubscriber; +//! +//! #[async_trait] +//! impl BuildEventSubscriber for MySubscriber { +//! async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { +//! // Handle event (serialize to protobuf, send over network, etc.) +//! println!("Event: {:?}", event); +//! Ok(()) +//! } +//! +//! fn name(&self) -> &str { +//! "MySubscriber" +//! } +//! } +//! ``` +//! +//! # Error Handling +//! +//! - **Bus emission**: Best-effort, never blocks. Slow subscribers may lag and miss events. +//! - **Subscriber errors**: Logged via `tracing::warn!`, but don't stop other subscribers. +//! - **Backpressure**: Broadcast channel size is configurable; slow subscribers will lag. +//! +//! # Future Extensions +//! +//! This crate provides pure Rust types. For protobuf support, create a separate +//! `build-events-proto` crate that converts these Rust types to protobuf messages +//! for gRPC streaming. + +pub mod bus; +pub mod dispatcher; +pub mod events; +pub mod subscriber; +pub mod subscribers; + +// Re-export main types for convenience +pub use bus::BuildEventBus; +pub use dispatcher::BuildEventDispatcher; +pub use events::BuildEvent; +pub use subscriber::{BuildEventSubscriber, SubscriberError}; diff --git a/crates/build-events/src/subscriber.rs b/crates/build-events/src/subscriber.rs new file mode 100644 index 0000000000..5803fd5f67 --- /dev/null +++ b/crates/build-events/src/subscriber.rs @@ -0,0 +1,126 @@ +//! Build event subscriber trait and error types +//! +//! Defines the [`BuildEventSubscriber`] trait that allows components to +//! receive and process build events. + +use crate::events::BuildEvent; +use async_trait::async_trait; +use thiserror::Error; + +/// Error type for subscriber operations +#[derive(Debug, Error)] +pub enum SubscriberError { + /// I/O error occurred during event processing + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + + /// Serialization error + #[error("Serialization error: {0}")] + Serialization(String), + + /// Custom error with a message + #[error("Subscriber error: {0}")] + Custom(String), +} + +/// Trait for components that want to receive build events +/// +/// Subscribers receive events through the [`on_event`](BuildEventSubscriber::on_event) +/// method and can perform any processing they need (logging, writing to files, +/// sending over network, etc.). +/// +/// Subscribers are responsible for their own serialization format - the +/// events are provided as strongly-typed Rust structs. +/// +/// # Example +/// ``` +/// use build_events::{BuildEvent, BuildEventSubscriber, SubscriberError}; +/// use async_trait::async_trait; +/// +/// struct MySubscriber; +/// +/// #[async_trait] +/// impl BuildEventSubscriber for MySubscriber { +/// async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { +/// println!("Received event: {:?}", event); +/// Ok(()) +/// } +/// +/// fn name(&self) -> &str { +/// "MySubscriber" +/// } +/// } +/// ``` +#[async_trait] +pub trait BuildEventSubscriber: Send + Sync { + /// Process a build event + /// + /// This method is called for each event emitted by the build. + /// Implementations should return quickly to avoid blocking the dispatcher. + /// + /// # Arguments + /// * `event` - The event to process + /// + /// # Errors + /// Returns a [`SubscriberError`] if event processing fails + async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError>; + + /// Get the name of this subscriber for identification + /// + /// Used in logging and debugging to identify which subscriber + /// is processing events. + fn name(&self) -> &str; + + /// Called when the subscriber is about to be dropped + /// + /// This is an optional hook that subscribers can use to perform + /// cleanup, flush buffers, close files, etc. + /// + /// The default implementation does nothing. + async fn on_close(&self) -> Result<(), SubscriberError> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::{BuildStarted, current_millis}; + + struct TestSubscriber { + name: String, + } + + #[async_trait] + impl BuildEventSubscriber for TestSubscriber { + async fn on_event(&self, _event: &BuildEvent) -> Result<(), SubscriberError> { + Ok(()) + } + + fn name(&self) -> &str { + &self.name + } + } + + #[tokio::test] + async fn test_subscriber_trait() { + let subscriber = TestSubscriber { + name: "test".to_string(), + }; + + assert_eq!(subscriber.name(), "test"); + + let event = BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + }); + + let result = subscriber.on_event(&event).await; + assert!(result.is_ok()); + + let close_result = subscriber.on_close().await; + assert!(close_result.is_ok()); + } +} diff --git a/crates/build-events/src/subscribers/json_file.rs b/crates/build-events/src/subscribers/json_file.rs new file mode 100644 index 0000000000..f610aab368 --- /dev/null +++ b/crates/build-events/src/subscribers/json_file.rs @@ -0,0 +1,204 @@ +//! JSON file writer subscriber implementation +//! +//! Writes build events to a file in JSON-lines format (one JSON object per line). + +use crate::events::BuildEvent; +use crate::subscriber::{BuildEventSubscriber, SubscriberError}; +use async_trait::async_trait; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +/// Subscriber that writes events to a JSON-lines file +/// +/// Each event is serialized to JSON and written as a single line to the file. +/// The file is automatically flushed after each write to ensure durability. +/// +/// # Example +/// ```no_run +/// use build_events::subscribers::JsonFileWriter; +/// +/// let writer = JsonFileWriter::new("build_events.jsonl").unwrap(); +/// ``` +pub struct JsonFileWriter { + file: Mutex, + path: PathBuf, +} + +impl JsonFileWriter { + /// Create a new JSON file writer + /// + /// Creates or truncates the file at the specified path. The file will + /// be written in JSON-lines format (one event per line). + /// + /// # Arguments + /// * `path` - Path to the output file + /// + /// # Errors + /// Returns an error if the file cannot be created or opened + /// + /// # Example + /// ```no_run + /// use build_events::subscribers::JsonFileWriter; + /// + /// let writer = JsonFileWriter::new("build_events.jsonl").unwrap(); + /// ``` + pub fn new>(path: P) -> std::io::Result { + let path = path.as_ref().to_path_buf(); + let file = std::fs::File::create(&path)?; + + Ok(Self { + file: Mutex::new(file), + path, + }) + } + + /// Get the path to the output file + pub fn path(&self) -> &Path { + &self.path + } +} + +impl std::fmt::Debug for JsonFileWriter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JsonFileWriter") + .field("path", &self.path) + .finish() + } +} + +#[async_trait] +impl BuildEventSubscriber for JsonFileWriter { + async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { + let json = serde_json::to_string(event).map_err(|e| { + SubscriberError::Serialization(format!("Failed to serialize event: {}", e)) + })?; + + let mut file = self.file.lock().unwrap(); + writeln!(file, "{}", json)?; + file.flush()?; + + Ok(()) + } + + fn name(&self) -> &str { + "JsonFileWriter" + } + + async fn on_close(&self) -> Result<(), SubscriberError> { + let mut file = self.file.lock().unwrap(); + file.flush()?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::{BuildStarted, TargetKind, TargetStarted, current_millis}; + use std::io::{BufRead, BufReader}; + + #[tokio::test] + async fn test_json_file_writer() { + let temp_dir = tempfile::tempdir().unwrap(); + let file_path = temp_dir.path().join("events.jsonl"); + + let writer = JsonFileWriter::new(&file_path).unwrap(); + assert_eq!(writer.name(), "JsonFileWriter"); + assert_eq!(writer.path(), file_path.as_path()); + + let event = BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command_line: vec!["cargo".to_string(), "build".to_string()], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + }); + + let result = writer.on_event(&event).await; + assert!(result.is_ok()); + + // Close and flush + writer.on_close().await.unwrap(); + drop(writer); + + // Read back and verify + let file = std::fs::File::open(&file_path).unwrap(); + let reader = BufReader::new(file); + let lines: Vec = reader.lines().map(|l| l.unwrap()).collect(); + + assert_eq!(lines.len(), 1); + + let deserialized: BuildEvent = serde_json::from_str(&lines[0]).unwrap(); + assert_eq!(deserialized, event); + } + + #[tokio::test] + async fn test_json_file_multiple_events() { + let temp_dir = tempfile::tempdir().unwrap(); + let file_path = temp_dir.path().join("events.jsonl"); + + let writer = JsonFileWriter::new(&file_path).unwrap(); + + let events = vec![ + BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test-1".to_string(), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + }), + BuildEvent::TargetStarted(TargetStarted { + target_id: "target-1".to_string(), + label: "//foo:bar".to_string(), + target_kind: TargetKind::Binary, + timestamp_millis: current_millis(), + }), + ]; + + for event in &events { + writer.on_event(event).await.unwrap(); + } + + writer.on_close().await.unwrap(); + drop(writer); + + // Read back and verify + let file = std::fs::File::open(&file_path).unwrap(); + let reader = BufReader::new(file); + let lines: Vec = reader.lines().map(|l| l.unwrap()).collect(); + + assert_eq!(lines.len(), 2); + + for (i, line) in lines.iter().enumerate() { + let deserialized: BuildEvent = serde_json::from_str(line).unwrap(); + assert_eq!(deserialized, events[i]); + } + } + + #[tokio::test] + async fn test_json_file_truncates_existing() { + let temp_dir = tempfile::tempdir().unwrap(); + let file_path = temp_dir.path().join("events.jsonl"); + + // Write some initial data + std::fs::write(&file_path, "old data\n").unwrap(); + + // Create writer (should truncate) + let writer = JsonFileWriter::new(&file_path).unwrap(); + + let event = BuildEvent::BuildStarted(BuildStarted { + invocation_id: "new".to_string(), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + }); + + writer.on_event(&event).await.unwrap(); + writer.on_close().await.unwrap(); + drop(writer); + + // Read back - should only have new data + let content = std::fs::read_to_string(&file_path).unwrap(); + assert!(!content.contains("old data")); + assert!(content.contains("new")); + } +} diff --git a/crates/build-events/src/subscribers/logger.rs b/crates/build-events/src/subscribers/logger.rs new file mode 100644 index 0000000000..fc76e50279 --- /dev/null +++ b/crates/build-events/src/subscribers/logger.rs @@ -0,0 +1,165 @@ +//! Logger subscriber implementation +//! +//! Logs build events using the tracing crate with structured fields. + +use crate::events::BuildEvent; +use crate::subscriber::{BuildEventSubscriber, SubscriberError}; +use async_trait::async_trait; +use tracing::info; + +/// Subscriber that logs events using the tracing crate +/// +/// This subscriber is useful for debugging and development, as it +/// emits structured log entries for each build event. +/// +/// # Example +/// ``` +/// use build_events::subscribers::LoggerSubscriber; +/// +/// let subscriber = LoggerSubscriber::new(); +/// ``` +#[derive(Debug, Clone)] +pub struct LoggerSubscriber; + +impl LoggerSubscriber { + /// Create a new logger subscriber + /// + /// # Example + /// ``` + /// use build_events::subscribers::LoggerSubscriber; + /// + /// let subscriber = LoggerSubscriber::new(); + /// ``` + pub fn new() -> Self { + Self + } +} + +impl Default for LoggerSubscriber { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl BuildEventSubscriber for LoggerSubscriber { + async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { + match event { + BuildEvent::BuildStarted(e) => { + info!( + event = "build_started", + invocation_id = %e.invocation_id, + command_line = ?e.command_line, + timestamp_millis = e.timestamp_millis, + working_directory = %e.working_directory, + "Build started" + ); + } + BuildEvent::BuildFinished(e) => { + info!( + event = "build_finished", + invocation_id = %e.invocation_id, + success = e.success, + timestamp_millis = e.timestamp_millis, + error_message = ?e.error_message, + "Build finished" + ); + } + BuildEvent::TargetStarted(e) => { + info!( + event = "target_started", + label = %e.label, + target_kind = ?e.target_kind, + timestamp_millis = e.timestamp_millis, + "Target started" + ); + } + BuildEvent::TargetCompleted(e) => { + info!( + event = "target_completed", + label = %e.label, + success = e.success, + timestamp_millis = e.timestamp_millis, + error_message = ?e.error_message, + outputs = ?e.outputs, + "Target completed" + ); + } + BuildEvent::ActionStarted(e) => { + info!( + event = "action_started", + action_id = %e.action_id, + action_name = %e.action_name, + target_id = %e.target_id, + timestamp_millis = e.timestamp_millis, + "Action started" + ); + } + BuildEvent::ActionCompleted(e) => { + info!( + event = "action_completed", + action_id = %e.action_id, + success = e.success, + timestamp_millis = e.timestamp_millis, + exit_code = e.exit_code, + stdout = ?e.stdout, + stderr = ?e.stderr, + "Action completed" + ); + } + } + + Ok(()) + } + + fn name(&self) -> &str { + "LoggerSubscriber" + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::{BuildStarted, TargetKind, TargetStarted, current_millis}; + + #[tokio::test] + async fn test_logger_subscriber() { + let subscriber = LoggerSubscriber::new(); + assert_eq!(subscriber.name(), "LoggerSubscriber"); + + let event = BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command_line: vec!["cargo".to_string(), "build".to_string()], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + }); + + let result = subscriber.on_event(&event).await; + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_logger_all_event_types() { + let subscriber = LoggerSubscriber::new(); + + let events = vec![ + BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test".to_string(), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + }), + BuildEvent::TargetStarted(TargetStarted { + target_id: "target-1".to_string(), + label: "//foo:bar".to_string(), + target_kind: TargetKind::Binary, + timestamp_millis: current_millis(), + }), + ]; + + for event in events { + let result = subscriber.on_event(&event).await; + assert!(result.is_ok()); + } + } +} diff --git a/crates/build-events/src/subscribers/mod.rs b/crates/build-events/src/subscribers/mod.rs new file mode 100644 index 0000000000..a84da1d825 --- /dev/null +++ b/crates/build-events/src/subscribers/mod.rs @@ -0,0 +1,11 @@ +//! Built-in subscriber implementations +//! +//! This module provides ready-to-use subscriber implementations: +//! - [`LoggerSubscriber`]: Logs events using tracing +//! - [`JsonFileWriter`]: Writes events to a JSON-lines file + +mod json_file; +mod logger; + +pub use json_file::JsonFileWriter; +pub use logger::LoggerSubscriber; diff --git a/crates/build-events/tests/integration_test.rs b/crates/build-events/tests/integration_test.rs new file mode 100644 index 0000000000..995b3d6efb --- /dev/null +++ b/crates/build-events/tests/integration_test.rs @@ -0,0 +1,232 @@ +//! Integration tests for the build-events crate +//! +//! Tests the full end-to-end flow of event emission, distribution, and subscription. + +use async_trait::async_trait; +use build_events::{ + BuildEvent, BuildEventBus, BuildEventDispatcher, BuildEventSubscriber, SubscriberError, + events::{ + ActionCompleted, ActionStarted, BuildFinished, BuildStarted, TargetCompleted, TargetKind, + TargetStarted, current_millis, + }, + subscribers::{JsonFileWriter, LoggerSubscriber}, +}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// Test subscriber that counts events +struct CountingSubscriber { + count: Arc, +} + +#[async_trait] +impl BuildEventSubscriber for CountingSubscriber { + async fn on_event(&self, _event: &BuildEvent) -> Result<(), SubscriberError> { + self.count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn name(&self) -> &str { + "CountingSubscriber" + } +} + +#[tokio::test] +async fn test_full_event_flow() { + let bus = BuildEventBus::new(100); + let count = Arc::new(AtomicUsize::new(0)); + + let mut dispatcher = BuildEventDispatcher::new(bus.subscribe()); + dispatcher.add_subscriber(Box::new(CountingSubscriber { + count: count.clone(), + })); + + // Start dispatcher + let handle = tokio::spawn(async move { + dispatcher.run().await; + }); + + // Emit all event types + bus.emit(BuildEvent::BuildStarted(BuildStarted { + invocation_id: "test-integration".to_string(), + command_line: vec!["cargo".to_string(), "test".to_string()], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })); + + bus.emit(BuildEvent::TargetStarted(TargetStarted { + target_id: "target-1".to_string(), + label: "//foo:bar".to_string(), + target_kind: TargetKind::Binary, + timestamp_millis: current_millis(), + })); + + bus.emit(BuildEvent::ActionStarted(ActionStarted { + action_id: "action-1".to_string(), + action_name: "CppCompile".to_string(), + target_id: "target-1".to_string(), + timestamp_millis: current_millis(), + })); + + bus.emit(BuildEvent::ActionCompleted(ActionCompleted { + action_id: "action-1".to_string(), + success: true, + timestamp_millis: current_millis(), + exit_code: 0, + stdout: Some("compiled successfully".to_string()), + stderr: None, + })); + + bus.emit(BuildEvent::TargetCompleted(TargetCompleted { + target_id: "target-1".to_string(), + label: "//foo:bar".to_string(), + success: true, + timestamp_millis: current_millis(), + error_message: None, + outputs: vec!["foo/bar".to_string()], + cache_hit: false, + })); + + bus.emit(BuildEvent::BuildFinished(BuildFinished { + invocation_id: "test-integration".to_string(), + success: true, + timestamp_millis: current_millis(), + error_message: None, + })); + + // Wait for events to process + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + // Close bus and wait for dispatcher + drop(bus); + handle.await.unwrap(); + + // Verify all events were received + assert_eq!(count.load(Ordering::SeqCst), 6); +} + +#[tokio::test] +async fn test_multiple_subscribers() { + let temp_dir = tempfile::tempdir().unwrap(); + let json_path = temp_dir.path().join("events.jsonl"); + + let bus = BuildEventBus::new(100); + let count = Arc::new(AtomicUsize::new(0)); + + let mut dispatcher = BuildEventDispatcher::new(bus.subscribe()); + dispatcher.add_subscriber(Box::new(LoggerSubscriber::new())); + dispatcher.add_subscriber(Box::new(JsonFileWriter::new(&json_path).unwrap())); + dispatcher.add_subscriber(Box::new(CountingSubscriber { + count: count.clone(), + })); + + let handle = tokio::spawn(async move { + dispatcher.run().await; + }); + + // Emit a few events + for i in 0..3 { + bus.emit(BuildEvent::BuildStarted(BuildStarted { + invocation_id: format!("test-{}", i), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })); + } + + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + drop(bus); + handle.await.unwrap(); + + // Verify counter subscriber received events + assert_eq!(count.load(Ordering::SeqCst), 3); + + // Verify JSON file has events + let content = std::fs::read_to_string(&json_path).unwrap(); + let lines: Vec<&str> = content.lines().collect(); + assert_eq!(lines.len(), 3); + + // Verify each line is valid JSON + for line in lines { + let _: BuildEvent = serde_json::from_str(line).unwrap(); + } +} + +#[tokio::test] +async fn test_late_subscriber_misses_early_events() { + let bus = BuildEventBus::new(100); + + // Emit event before subscription + bus.emit(BuildEvent::BuildStarted(BuildStarted { + invocation_id: "early".to_string(), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })); + + // Subscribe after emission + let count = Arc::new(AtomicUsize::new(0)); + let mut dispatcher = BuildEventDispatcher::new(bus.subscribe()); + dispatcher.add_subscriber(Box::new(CountingSubscriber { + count: count.clone(), + })); + + let handle = tokio::spawn(async move { + dispatcher.run().await; + }); + + // Emit event after subscription + bus.emit(BuildEvent::BuildStarted(BuildStarted { + invocation_id: "late".to_string(), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })); + + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + drop(bus); + handle.await.unwrap(); + + // Should only have received the late event + assert_eq!(count.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn test_bus_cloning() { + let bus = BuildEventBus::new(100); + let bus_clone = bus.clone(); + let count = Arc::new(AtomicUsize::new(0)); + + let mut dispatcher = BuildEventDispatcher::new(bus.subscribe()); + dispatcher.add_subscriber(Box::new(CountingSubscriber { + count: count.clone(), + })); + + let handle = tokio::spawn(async move { + dispatcher.run().await; + }); + + // Emit from original bus + bus.emit(BuildEvent::BuildStarted(BuildStarted { + invocation_id: "original".to_string(), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })); + + // Emit from cloned bus + bus_clone.emit(BuildEvent::BuildStarted(BuildStarted { + invocation_id: "clone".to_string(), + command_line: vec![], + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })); + + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + drop(bus); + drop(bus_clone); + handle.await.unwrap(); + + // Both events should be received + assert_eq!(count.load(Ordering::SeqCst), 2); +} diff --git a/crates/minimal/Cargo.toml b/crates/minimal/Cargo.toml index 9d612406b4..bc4c9ec89c 100644 --- a/crates/minimal/Cargo.toml +++ b/crates/minimal/Cargo.toml @@ -7,6 +7,8 @@ publish.workspace = true [dependencies] "build-sandbox".workspace = true spongebob = { path = "../spongebob" } +build-events.workspace = true +build-events-proto = { path = "../build-events-proto", features = ["spongebob-subscriber"] } graph.workspace = true cache.workspace = true common.workspace = true diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index 1993009617..b71ddd13cf 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -1,6 +1,9 @@ use crate::{Error, lockfile::PrebuiltsLock, remote_storage::RemoteStorage, run::Run}; use crate::{GlobalArgs, PackagesArg}; use anyhow::Context; +use build_events::events::{BuildEvent, BuildFinished, BuildStarted, current_millis}; +use build_events::{BuildEventBus, BuildEventDispatcher}; +use build_events_proto::SpongeBobSubscriberV2; use cache::{Cache, CacheBinProvider, LocalDir, RemoteBinProvider}; use graph::{DepGraph, ExecPlan, Transitives}; use std::path::Path; @@ -32,15 +35,7 @@ pub async fn cmd_build_impl( .await .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; - let command_info = format!( - "build --packages {}", - graph - .top_levels - .iter() - .map(|bsr| graph.get(bsr).unwrap().name.as_str()) - .collect::>() - .join(",") - ); + let mut spongebob_invocation = Some(spongebob_client.create_invocation()); rayon::ThreadPoolBuilder::new() .num_threads(num_parallel_builds) @@ -60,15 +55,43 @@ pub async fn cmd_build_impl( output_base, ); - let mut spongebob_invocation = match spongebob_client.create_invocation(&command_info).await { - Ok(inv) => Some(inv), - Err(e) => { - tracing::warn!("Failed to create SpongeBob invocation: {}", e); - None - } - }; + // Setup build events system + let event_bus = BuildEventBus::new(10000); + + // Create dispatcher with SpongeBobSubscriberV2 if we have an invocation + if let Some(ref invocation) = spongebob_invocation { + let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); + let subscriber = SpongeBobSubscriberV2::from_invocation(invocation.clone()); + dispatcher.add_subscriber(Box::new(subscriber)); + + // Spawn dispatcher in background + tokio::spawn(async move { + dispatcher.run().await; + }); + } + + // Get invocation_id for events + let invocation_id = spongebob_invocation + .as_ref() + .map(|inv| inv.invocation_id().to_string()) + .unwrap_or_else(|| "local".to_string()); - match (globals.no_cache, globals.no_fetch) { + // Get command line for BuildStarted event + let command_line = std::env::args().collect::>(); + let working_directory = std::env::current_dir() + .unwrap_or_default() + .to_string_lossy() + .to_string(); + + // Emit BuildStarted event + event_bus.emit(BuildEvent::BuildStarted(BuildStarted { + invocation_id: invocation_id.clone(), + command_line, + timestamp_millis: current_millis(), + working_directory, + })); + + let build_success = match (globals.no_cache, globals.no_fetch) { // No local or remote cache (true, true) => { run.execute(ExecPlan::new(graph), None, &mut spongebob_invocation) @@ -107,8 +130,22 @@ pub async fn cmd_build_impl( ) .await } - } - .context("Failed to execute build")?; + }; + + // Determine if build was successful + let build_succeeded = build_success.is_ok(); + let error_message = build_success.as_ref().err().map(|e| e.to_string()); + + // Emit BuildFinished event + event_bus.emit(BuildEvent::BuildFinished(BuildFinished { + invocation_id: invocation_id.clone(), + success: build_succeeded, + timestamp_millis: current_millis(), + error_message, + })); + + // Propagate error if build failed + build_success.context("Failed to execute build")?; // If we got this far, everything we need is either fetchable or built. // @@ -162,7 +199,7 @@ pub async fn cmd_build_impl( // Log build results to SpongeBob if let Some(ref mut invocation) = spongebob_invocation { - log_build_results_to_spongebob(invocation, graph, true).await?; + log_build_results_to_spongebob(invocation, graph, build_succeeded).await?; } let spongebob_url = spongebob_invocation diff --git a/crates/minimal/src/cmd_unsafe_patched_build.rs b/crates/minimal/src/cmd_unsafe_patched_build.rs index 2094f90779..d795d94a76 100644 --- a/crates/minimal/src/cmd_unsafe_patched_build.rs +++ b/crates/minimal/src/cmd_unsafe_patched_build.rs @@ -86,15 +86,11 @@ pub async fn cmd_unsafe_patched_build( .unwrap(); info!("Building package: {}", build.name); - let command_info = format!("unsafe-patched-build {}", build.name); - let mut spongebob_invocation = Some( - spongebob::SpongeBob::new() - .await - .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))? - .create_invocation(&command_info) - .await - .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob invocation: {}", e))?, - ); + let mut spongebob_client = spongebob::SpongeBob::new() + .await + .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; + + let mut spongebob_invocation = Some(spongebob_client.create_invocation()); run_build( &config, out_dir.path(), diff --git a/crates/spongebob/Cargo.toml b/crates/spongebob/Cargo.toml index c389d8f9f6..66b524aba2 100644 --- a/crates/spongebob/Cargo.toml +++ b/crates/spongebob/Cargo.toml @@ -5,12 +5,9 @@ edition.workspace = true publish.workspace = true [dependencies] -minimal_spongebob_community_neoeinstein-prost = { version = "0.4.0-20251007185801-98893cf03142.1", registry = "buf" } -minimal_spongebob_community_neoeinstein-tonic = { version = "0.4.1-20251007185801-98893cf03142.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251008233118-4a2c40ff6cb8.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251008233118-4a2c40ff6cb8.1", registry = "buf" } thiserror.workspace = true -tonic = "0.12.3" +tonic = { version = "0.12.3", features = ["tls", "tls-native-roots"] } tracing.workspace = true uuid.workspace = true - -[build-dependencies] -tonic-prost-build.workspace = true diff --git a/crates/spongebob/src/lib.rs b/crates/spongebob/src/lib.rs index e079e283dd..56c146bf88 100644 --- a/crates/spongebob/src/lib.rs +++ b/crates/spongebob/src/lib.rs @@ -1,5 +1,10 @@ -use minimal_spongebob_community_neoeinstein_prost::spongebob::v1::{CreateFileRequest, CreateInvocationRequest, File, Invocation}; -use minimal_spongebob_community_neoeinstein_tonic::spongebob::v1::tonic::sponge_bob_service_client::SpongeBobServiceClient; +use minimal_spongebob_community_neoeinstein_prost::spongebob::v1::{ + BuildEvent, CreateFileRequest, File, PublishBuildEventRequest, +}; +use minimal_spongebob_community_neoeinstein_tonic::spongebob::v1::tonic::{ + build_event_service_client::BuildEventServiceClient, + result_storage_service_client::ResultStorageServiceClient, +}; use tonic::transport::Channel; use uuid::Uuid; @@ -17,22 +22,43 @@ pub type Result = std::result::Result; #[derive(Debug)] pub struct SpongeBob { - service: SpongeBobServiceClient, + build_event_service: BuildEventServiceClient, + result_storage_service: ResultStorageServiceClient, } #[derive(Debug, Clone)] pub struct SpongeBobInvocation { - service: SpongeBobServiceClient, - resource_name: String, + build_event_service: BuildEventServiceClient, + result_storage_service: ResultStorageServiceClient, + invocation_id: String, url: String, } impl SpongeBobInvocation { + /// Create a new invocation (no RPC needed - invocation created on first BuildStarted event) + pub fn new( + build_event_service: BuildEventServiceClient, + result_storage_service: ResultStorageServiceClient, + invocation_id: String, + ) -> Self { + let url = format!( + "{}/invocations/{}", + "https://dash.minimal.dev", invocation_id + ); + Self { + build_event_service, + result_storage_service, + invocation_id, + url, + } + } + #[tracing::instrument(skip(self, contents))] pub async fn upload_file(&mut self, file_name: &str, contents: Vec) -> Result<()> { let file_id = Uuid::new_v4().to_string(); + let parent = format!("invocations/{}", self.invocation_id); let request = CreateFileRequest { - parent: self.resource_name.clone(), + parent, file_id, file: Some(File { name: file_name.to_string(), @@ -40,7 +66,22 @@ impl SpongeBobInvocation { }), }; - self.service.create_file(request).await?; + self.result_storage_service.create_file(request).await?; + Ok(()) + } + + /// Publish a build event to Spongebob + /// Invocation is auto-created on first BuildStarted event + #[tracing::instrument(skip(self, event))] + pub async fn publish_build_event(&mut self, event: BuildEvent) -> Result<()> { + let request = PublishBuildEventRequest { + event: Some(event), + request_id: String::new(), // Optional idempotency key + }; + + self.build_event_service + .publish_build_event(request) + .await?; Ok(()) } @@ -48,55 +89,33 @@ impl SpongeBobInvocation { &self.url } - pub fn resource_name(&self) -> &str { - &self.resource_name + pub fn invocation_id(&self) -> &str { + &self.invocation_id + } + + pub fn resource_name(&self) -> String { + format!("invocations/{}", self.invocation_id) } } impl SpongeBob { pub async fn new() -> Result { - let service = SpongeBobServiceClient::connect(ENDPOINT).await?; + let build_event_service = BuildEventServiceClient::connect(ENDPOINT).await?; + let result_storage_service = ResultStorageServiceClient::connect(ENDPOINT).await?; - Ok(SpongeBob { service }) - } - - #[tracing::instrument] - pub async fn create_invocation(&mut self, name: &str) -> Result { - let invocation_id = Self::generate_invocation_id(name); - let resource_name = self.create_invocation_with_id(name, &invocation_id).await?; - let url = self.generate_invocation_url(&invocation_id); - - Ok(SpongeBobInvocation { - service: self.service.clone(), - resource_name, - url, + Ok(SpongeBob { + build_event_service, + result_storage_service, }) } - #[tracing::instrument] - async fn create_invocation_with_id( - &mut self, - name: &str, - invocation_id: &str, - ) -> Result { - let request = CreateInvocationRequest { - invocation_id: invocation_id.to_string(), - invocation: Some(Invocation { - name: name.to_string(), - }), - }; - - let response = self.service.create_invocation(request).await?; - Ok(response.into_inner().name) - } - - fn generate_invocation_id(_name: &str) -> String { - // Just use a UUID for the invocation ID to ensure URL safety - Uuid::new_v4().to_string() - } - - /// Generate the web URL for viewing an invocation - fn generate_invocation_url(&self, invocation_id: &str) -> String { - format!("{}/invocation/{}", ENDPOINT, invocation_id) + /// Create a new invocation handle (no RPC - invocation created on first BuildStarted event) + pub fn create_invocation(&mut self) -> SpongeBobInvocation { + let invocation_id = Uuid::new_v4().to_string(); + SpongeBobInvocation::new( + self.build_event_service.clone(), + self.result_storage_service.clone(), + invocation_id, + ) } } From 33837824bef044ed500b69a817bd615f8cd8abe4 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Thu, 9 Oct 2025 11:44:11 -0700 Subject: [PATCH 03/39] more vibes --- Cargo.lock | 14 +++++++------- crates/build-events-proto/Cargo.toml | 2 +- crates/build-events-proto/proto/events.proto | 2 +- crates/build-events-proto/src/convert.rs | 8 ++++---- crates/build-events-proto/src/lib.rs | 2 +- crates/build-events-proto/src/spongebob_convert.rs | 6 +++--- crates/build-events-proto/src/subscriber.rs | 2 +- crates/build-events/examples/basic_usage.rs | 6 +----- crates/build-events/src/bus.rs | 14 +++++++------- crates/build-events/src/dispatcher.rs | 6 +++--- crates/build-events/src/events.rs | 6 +++--- crates/build-events/src/lib.rs | 1 - crates/build-events/src/subscriber.rs | 2 +- crates/build-events/src/subscribers/json_file.rs | 6 +++--- crates/build-events/src/subscribers/logger.rs | 6 +++--- crates/build-events/tests/integration_test.rs | 12 ++++++------ crates/minimal/src/cmd_build.rs | 12 +++++++++--- crates/spongebob/Cargo.toml | 4 ++-- 18 files changed, 56 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 619251f37e..c9e7c02b7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2276,9 +2276,9 @@ dependencies = [ [[package]] name = "minimal_spongebob_community_neoeinstein-prost" -version = "0.4.0-20251008233118-4a2c40ff6cb8.1" +version = "0.4.0-20251009015156-8bcd87750609.1" source = "sparse+https://buf.build/gen/cargo/" -checksum = "3b7f67266fb3637565abb8b4513f7ac5b6b1db85ef32b52378490b32a5f1cf6d" +checksum = "699b4089a62d6370864b932906ff4ba10eec4485cf9ea0ef3d868dd8352b5e4f" dependencies = [ "prost 0.13.5", "prost-types 0.13.5", @@ -2286,9 +2286,9 @@ dependencies = [ [[package]] name = "minimal_spongebob_community_neoeinstein-tonic" -version = "0.4.1-20251008233118-4a2c40ff6cb8.1" +version = "0.4.1-20251009015156-8bcd87750609.1" source = "sparse+https://buf.build/gen/cargo/" -checksum = "243b90d6da5fd10bfd4ce7da52788eece61c44d8d5847ca2d0346d435b7ea8af" +checksum = "fa73a8386302f7d6aba6c519bbaacb9087e711d79d9496ec191b2c22bb2627b4" dependencies = [ "minimal_spongebob_community_neoeinstein-prost", "tonic 0.12.3", @@ -2868,7 +2868,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" dependencies = [ "heck 0.5.0", - "itertools 0.11.0", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -2890,7 +2890,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -2903,7 +2903,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn", diff --git a/crates/build-events-proto/Cargo.toml b/crates/build-events-proto/Cargo.toml index f6d08e3783..8d09cbcd5d 100644 --- a/crates/build-events-proto/Cargo.toml +++ b/crates/build-events-proto/Cargo.toml @@ -21,7 +21,7 @@ tokio-stream = { version = "0.1", features = ["sync"] } spongebob = { path = "../spongebob", optional = true } # Need proto types for conversion when spongebob-subscriber feature is enabled -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251008233118-4a2c40ff6cb8.1", registry = "buf", optional = true } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251009015156-8bcd87750609.1", registry = "buf", optional = true } [features] default = [] diff --git a/crates/build-events-proto/proto/events.proto b/crates/build-events-proto/proto/events.proto index 40472b8f82..2a2d271de0 100644 --- a/crates/build-events-proto/proto/events.proto +++ b/crates/build-events-proto/proto/events.proto @@ -17,9 +17,9 @@ message BuildEvent { // BuildStarted is emitted when a build invocation starts message BuildStarted { string invocation_id = 1; - repeated string command_line = 2; int64 timestamp_millis = 3; string working_directory = 4; + string command = 5; // Command string for display } // BuildFinished is emitted when a build invocation completes diff --git a/crates/build-events-proto/src/convert.rs b/crates/build-events-proto/src/convert.rs index 862aaad61d..e41a888724 100644 --- a/crates/build-events-proto/src/convert.rs +++ b/crates/build-events-proto/src/convert.rs @@ -41,7 +41,7 @@ impl From for proto::BuildStarted { fn from(event: events::BuildStarted) -> Self { proto::BuildStarted { invocation_id: event.invocation_id, - command_line: event.command_line, + command: event.command, timestamp_millis: event.timestamp_millis, working_directory: event.working_directory, } @@ -157,7 +157,7 @@ impl TryFrom for events::BuildStarted { fn try_from(event: proto::BuildStarted) -> Result { Ok(events::BuildStarted { invocation_id: event.invocation_id, - command_line: event.command_line, + command: event.command, timestamp_millis: event.timestamp_millis, working_directory: event.working_directory, }) @@ -272,7 +272,7 @@ mod tests { fn test_build_started_round_trip() { let rust_event = events::BuildStarted { invocation_id: "test-123".to_string(), - command_line: vec!["cargo".to_string(), "build".to_string()], + command: "build".to_string(), timestamp_millis: 1234567890, working_directory: "/tmp".to_string(), }; @@ -287,7 +287,7 @@ mod tests { fn test_build_event_round_trip() { let rust_event = events::BuildEvent::BuildStarted(events::BuildStarted { invocation_id: "test-456".to_string(), - command_line: vec!["cargo".to_string(), "test".to_string()], + command: "test".to_string(), timestamp_millis: 9876543210, working_directory: "/home/user".to_string(), }); diff --git a/crates/build-events-proto/src/lib.rs b/crates/build-events-proto/src/lib.rs index 35ac324fa2..dfbcca99b4 100644 --- a/crates/build-events-proto/src/lib.rs +++ b/crates/build-events-proto/src/lib.rs @@ -101,7 +101,7 @@ //! // Rust -> Proto //! let rust_event = BuildEvent::BuildStarted(BuildStarted { //! invocation_id: "test-123".to_string(), -//! command_line: vec!["cargo".to_string(), "build".to_string()], +//! command: "build".to_string(), //! timestamp_millis: current_millis(), //! working_directory: "/tmp".to_string(), //! }); diff --git a/crates/build-events-proto/src/spongebob_convert.rs b/crates/build-events-proto/src/spongebob_convert.rs index 2918c8dc5a..5f58d3ecc8 100644 --- a/crates/build-events-proto/src/spongebob_convert.rs +++ b/crates/build-events-proto/src/spongebob_convert.rs @@ -57,7 +57,7 @@ pub fn to_proto_build_event(event: &BuildEvent, invocation_id: &str) -> pb::Buil fn to_proto_build_started(e: &BuildStarted) -> pb::BuildStarted { pb::BuildStarted { invocation_id: e.invocation_id.clone(), - command_line: e.command_line.clone(), + command: e.command.clone(), timestamp_millis: e.timestamp_millis, working_directory: e.working_directory.clone(), } @@ -130,7 +130,7 @@ mod tests { fn test_build_started_conversion() { let rust_event = BuildEvent::BuildStarted(BuildStarted { invocation_id: "test-123".to_string(), - command_line: vec!["cargo".to_string(), "build".to_string()], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), }); @@ -141,7 +141,7 @@ mod tests { match proto_event.event.unwrap() { pb::build_event::Event::BuildStarted(e) => { assert_eq!(e.invocation_id, "test-123"); - assert_eq!(e.command_line.len(), 2); + assert_eq!(e.command, "build"); } _ => panic!("Wrong event type"), } diff --git a/crates/build-events-proto/src/subscriber.rs b/crates/build-events-proto/src/subscriber.rs index 2abf8f330c..417d0f8ee0 100644 --- a/crates/build-events-proto/src/subscriber.rs +++ b/crates/build-events-proto/src/subscriber.rs @@ -130,7 +130,7 @@ mod tests { let event = BuildEvent::BuildStarted(BuildStarted { invocation_id: "test-123".to_string(), - command_line: vec!["cargo".to_string(), "test".to_string()], + command: "test".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), }); diff --git a/crates/build-events/examples/basic_usage.rs b/crates/build-events/examples/basic_usage.rs index 3697824d18..49f6567990 100644 --- a/crates/build-events/examples/basic_usage.rs +++ b/crates/build-events/examples/basic_usage.rs @@ -77,11 +77,7 @@ async fn simulate_build(bus: &BuildEventBus) { println!("1. Emitting BuildStarted event"); bus.emit(BuildEvent::BuildStarted(BuildStarted { invocation_id: "demo-invocation-123".to_string(), - command_line: vec![ - "minimal".to_string(), - "build".to_string(), - "//example:target".to_string(), - ], + command: "build //example:target".to_string(), timestamp_millis: current_millis(), working_directory: std::env::current_dir() .unwrap() diff --git a/crates/build-events/src/bus.rs b/crates/build-events/src/bus.rs index 264900cf32..aa0ba57606 100644 --- a/crates/build-events/src/bus.rs +++ b/crates/build-events/src/bus.rs @@ -24,7 +24,7 @@ use tokio::sync::broadcast; /// /// bus.emit(BuildEvent::BuildStarted(BuildStarted { /// invocation_id: "test-123".to_string(), -/// command_line: vec!["cargo".to_string(), "build".to_string()], +/// command: "build".to_string(), /// timestamp_millis: current_millis(), /// working_directory: "/tmp".to_string(), /// })); @@ -79,7 +79,7 @@ impl BuildEventBus { /// let bus = BuildEventBus::new(1000); /// bus.emit(BuildEvent::BuildStarted(BuildStarted { /// invocation_id: "test-123".to_string(), - /// command_line: vec!["cargo".to_string(), "build".to_string()], + /// command: "build".to_string(), /// timestamp_millis: current_millis(), /// working_directory: "/tmp".to_string(), /// })); @@ -138,7 +138,7 @@ mod tests { let event = BuildEvent::BuildStarted(BuildStarted { invocation_id: "test-123".to_string(), - command_line: vec!["cargo".to_string(), "build".to_string()], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), }); @@ -157,7 +157,7 @@ mod tests { let event = BuildEvent::BuildStarted(BuildStarted { invocation_id: "test-456".to_string(), - command_line: vec!["cargo".to_string(), "test".to_string()], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), }); @@ -178,7 +178,7 @@ mod tests { let event = BuildEvent::BuildStarted(BuildStarted { invocation_id: "test-789".to_string(), - command_line: vec!["cargo".to_string(), "run".to_string()], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), }); @@ -208,7 +208,7 @@ mod tests { let event1 = BuildEvent::BuildStarted(BuildStarted { invocation_id: "early".to_string(), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), }); @@ -221,7 +221,7 @@ mod tests { let event2 = BuildEvent::BuildStarted(BuildStarted { invocation_id: "late".to_string(), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), }); diff --git a/crates/build-events/src/dispatcher.rs b/crates/build-events/src/dispatcher.rs index a81302662b..6e62b4366c 100644 --- a/crates/build-events/src/dispatcher.rs +++ b/crates/build-events/src/dispatcher.rs @@ -192,7 +192,7 @@ mod tests { sender .send(BuildEvent::BuildStarted(BuildStarted { invocation_id: format!("test-{}", i), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), })) @@ -236,7 +236,7 @@ mod tests { sender .send(BuildEvent::BuildStarted(BuildStarted { invocation_id: format!("test-{}", i), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), })) @@ -290,7 +290,7 @@ mod tests { sender .send(BuildEvent::BuildStarted(BuildStarted { invocation_id: "test".to_string(), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), })) diff --git a/crates/build-events/src/events.rs b/crates/build-events/src/events.rs index f601a035c5..df7ec2be17 100644 --- a/crates/build-events/src/events.rs +++ b/crates/build-events/src/events.rs @@ -29,8 +29,8 @@ pub enum BuildEvent { pub struct BuildStarted { /// Unique identifier for this build invocation pub invocation_id: String, - /// Command line arguments used to start the build - pub command_line: Vec, + /// Command string for display + pub command: String, /// Unix timestamp in milliseconds pub timestamp_millis: i64, /// Working directory where build was started @@ -156,7 +156,7 @@ mod tests { fn test_event_serialization() { let event = BuildEvent::BuildStarted(BuildStarted { invocation_id: "test-123".to_string(), - command_line: vec!["cargo".to_string(), "build".to_string()], + command: "build".to_string(), timestamp_millis: 1234567890, working_directory: "/tmp".to_string(), }); diff --git a/crates/build-events/src/lib.rs b/crates/build-events/src/lib.rs index b40b0a8def..b5ff600919 100644 --- a/crates/build-events/src/lib.rs +++ b/crates/build-events/src/lib.rs @@ -54,7 +54,6 @@ //! // Emit events from build //! event_bus.emit(BuildEvent::BuildStarted(BuildStarted { //! invocation_id: "build-123".to_string(), -//! command_line: std::env::args().collect(), //! timestamp_millis: current_millis(), //! working_directory: std::env::current_dir() //! .unwrap() diff --git a/crates/build-events/src/subscriber.rs b/crates/build-events/src/subscriber.rs index 5803fd5f67..c53e7f00d6 100644 --- a/crates/build-events/src/subscriber.rs +++ b/crates/build-events/src/subscriber.rs @@ -112,7 +112,7 @@ mod tests { let event = BuildEvent::BuildStarted(BuildStarted { invocation_id: "test-123".to_string(), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), }); diff --git a/crates/build-events/src/subscribers/json_file.rs b/crates/build-events/src/subscribers/json_file.rs index f610aab368..dc0e085633 100644 --- a/crates/build-events/src/subscribers/json_file.rs +++ b/crates/build-events/src/subscribers/json_file.rs @@ -109,7 +109,7 @@ mod tests { let event = BuildEvent::BuildStarted(BuildStarted { invocation_id: "test-123".to_string(), - command_line: vec!["cargo".to_string(), "build".to_string()], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), }); @@ -142,7 +142,7 @@ mod tests { let events = vec![ BuildEvent::BuildStarted(BuildStarted { invocation_id: "test-1".to_string(), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), }), @@ -187,7 +187,7 @@ mod tests { let event = BuildEvent::BuildStarted(BuildStarted { invocation_id: "new".to_string(), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), }); diff --git a/crates/build-events/src/subscribers/logger.rs b/crates/build-events/src/subscribers/logger.rs index fc76e50279..d8af556794 100644 --- a/crates/build-events/src/subscribers/logger.rs +++ b/crates/build-events/src/subscribers/logger.rs @@ -49,7 +49,7 @@ impl BuildEventSubscriber for LoggerSubscriber { info!( event = "build_started", invocation_id = %e.invocation_id, - command_line = ?e.command_line, + command = %e.command, timestamp_millis = e.timestamp_millis, working_directory = %e.working_directory, "Build started" @@ -129,7 +129,7 @@ mod tests { let event = BuildEvent::BuildStarted(BuildStarted { invocation_id: "test-123".to_string(), - command_line: vec!["cargo".to_string(), "build".to_string()], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), }); @@ -145,7 +145,7 @@ mod tests { let events = vec![ BuildEvent::BuildStarted(BuildStarted { invocation_id: "test".to_string(), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), }), diff --git a/crates/build-events/tests/integration_test.rs b/crates/build-events/tests/integration_test.rs index 995b3d6efb..ef2fa202f2 100644 --- a/crates/build-events/tests/integration_test.rs +++ b/crates/build-events/tests/integration_test.rs @@ -49,7 +49,7 @@ async fn test_full_event_flow() { // Emit all event types bus.emit(BuildEvent::BuildStarted(BuildStarted { invocation_id: "test-integration".to_string(), - command_line: vec!["cargo".to_string(), "test".to_string()], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), })); @@ -128,7 +128,7 @@ async fn test_multiple_subscribers() { for i in 0..3 { bus.emit(BuildEvent::BuildStarted(BuildStarted { invocation_id: format!("test-{}", i), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), })); @@ -159,7 +159,7 @@ async fn test_late_subscriber_misses_early_events() { // Emit event before subscription bus.emit(BuildEvent::BuildStarted(BuildStarted { invocation_id: "early".to_string(), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), })); @@ -178,7 +178,7 @@ async fn test_late_subscriber_misses_early_events() { // Emit event after subscription bus.emit(BuildEvent::BuildStarted(BuildStarted { invocation_id: "late".to_string(), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), })); @@ -209,7 +209,7 @@ async fn test_bus_cloning() { // Emit from original bus bus.emit(BuildEvent::BuildStarted(BuildStarted { invocation_id: "original".to_string(), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), })); @@ -217,7 +217,7 @@ async fn test_bus_cloning() { // Emit from cloned bus bus_clone.emit(BuildEvent::BuildStarted(BuildStarted { invocation_id: "clone".to_string(), - command_line: vec![], + command: "build".to_string(), timestamp_millis: current_millis(), working_directory: "/tmp".to_string(), })); diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index b71ddd13cf..e646e5a56f 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -76,8 +76,14 @@ pub async fn cmd_build_impl( .map(|inv| inv.invocation_id().to_string()) .unwrap_or_else(|| "local".to_string()); - // Get command line for BuildStarted event - let command_line = std::env::args().collect::>(); + // Get command for BuildStarted event (skip binary path at index 0) + let args: Vec = std::env::args().collect(); + let command = if args.len() > 1 { + args[1..].join(" ") + } else { + args.first().cloned().unwrap_or_default() + }; + let working_directory = std::env::current_dir() .unwrap_or_default() .to_string_lossy() @@ -86,7 +92,7 @@ pub async fn cmd_build_impl( // Emit BuildStarted event event_bus.emit(BuildEvent::BuildStarted(BuildStarted { invocation_id: invocation_id.clone(), - command_line, + command, timestamp_millis: current_millis(), working_directory, })); diff --git a/crates/spongebob/Cargo.toml b/crates/spongebob/Cargo.toml index 66b524aba2..346a22972b 100644 --- a/crates/spongebob/Cargo.toml +++ b/crates/spongebob/Cargo.toml @@ -5,8 +5,8 @@ edition.workspace = true publish.workspace = true [dependencies] -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251008233118-4a2c40ff6cb8.1", registry = "buf" } -minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251008233118-4a2c40ff6cb8.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251009015156-8bcd87750609.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251009015156-8bcd87750609.1", registry = "buf" } thiserror.workspace = true tonic = { version = "0.12.3", features = ["tls", "tls-native-roots"] } tracing.workspace = true From fe8b2d51ff4ec0d73102dd9f17a7607530ecd2a8 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Thu, 9 Oct 2025 12:45:14 -0700 Subject: [PATCH 04/39] use canonical proto --- Cargo.lock | 141 +----- Cargo.toml | 7 +- crates/build-events-proto/Cargo.toml | 14 +- crates/build-events-proto/build.rs | 9 - crates/build-events-proto/proto/events.proto | 91 ---- crates/build-events-proto/src/convert.rs | 437 ++++++++++-------- crates/build-events-proto/src/lib.rs | 14 +- crates/build-events-proto/src/service.rs | 41 +- .../src/spongebob_convert.rs | 165 ------- .../src/spongebob_subscriber_v2.rs | 4 +- crates/build-events-proto/src/subscriber.rs | 15 +- 11 files changed, 307 insertions(+), 631 deletions(-) delete mode 100644 crates/build-events-proto/build.rs delete mode 100644 crates/build-events-proto/proto/events.proto delete mode 100644 crates/build-events-proto/src/spongebob_convert.rs diff --git a/Cargo.lock b/Cargo.lock index c9e7c02b7e..40eba8ab3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -182,14 +182,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", - "axum-core 0.4.5", + "axum-core", "bytes", "futures-util", "http", "http-body", "http-body-util", "itoa", - "matchit 0.7.3", + "matchit", "memchr", "mime", "percent-encoding", @@ -202,31 +202,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "axum" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871" -dependencies = [ - "axum-core 0.5.5", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "itoa", - "matchit 0.8.4", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde_core", - "sync_wrapper", - "tower 0.5.2", - "tower-layer", - "tower-service", -] - [[package]] name = "axum-core" version = "0.4.5" @@ -247,24 +222,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "axum-core" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "sync_wrapper", - "tower-layer", - "tower-service", -] - [[package]] name = "backtrace" version = "0.3.76" @@ -429,15 +386,13 @@ dependencies = [ "async-trait", "build-events", "minimal_spongebob_community_neoeinstein-prost", - "prost 0.14.1", + "minimal_spongebob_community_neoeinstein-tonic", "spongebob", "tempfile", "thiserror 2.0.17", "tokio", "tokio-stream", - "tonic 0.14.2", - "tonic-prost", - "tonic-prost-build", + "tonic 0.12.3", "tracing", "tracing-subscriber", ] @@ -2182,12 +2137,6 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" -[[package]] -name = "matchit" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" - [[package]] name = "md-5" version = "0.10.6" @@ -2315,12 +2264,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "multimap" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" - [[package]] name = "native-tls" version = "0.2.14" @@ -2861,28 +2804,6 @@ dependencies = [ "prost-derive 0.14.1", ] -[[package]] -name = "prost-build" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" -dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", - "log", - "multimap", - "once_cell", - "petgraph", - "prettyplease", - "prost 0.14.1", - "prost-types 0.14.1", - "pulldown-cmark", - "pulldown-cmark-to-cmark", - "regex", - "syn", - "tempfile", -] - [[package]] name = "prost-derive" version = "0.13.5" @@ -2927,26 +2848,6 @@ dependencies = [ "prost 0.14.1", ] -[[package]] -name = "pulldown-cmark" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" -dependencies = [ - "bitflags", - "memchr", - "unicase", -] - -[[package]] -name = "pulldown-cmark-to-cmark" -version = "21.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5b6a0769a491a08b31ea5c62494a8f144ee0987d86d670a8af4df1e1b7cde75" -dependencies = [ - "pulldown-cmark", -] - [[package]] name = "quinn" version = "0.11.9" @@ -4076,7 +3977,7 @@ checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ "async-stream", "async-trait", - "axum 0.7.9", + "axum", "base64 0.22.1", "bytes", "h2", @@ -4107,11 +4008,8 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" dependencies = [ - "async-trait", - "axum 0.8.6", "base64 0.22.1", "bytes", - "h2", "http", "http-body", "http-body-util", @@ -4121,7 +4019,6 @@ dependencies = [ "percent-encoding", "pin-project", "rustls-native-certs", - "socket2 0.6.0", "sync_wrapper", "tokio", "tokio-rustls", @@ -4132,18 +4029,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tonic-build" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c40aaccc9f9eccf2cd82ebc111adc13030d23e887244bc9cfa5d1d636049de3" -dependencies = [ - "prettyplease", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tonic-prost" version = "0.14.2" @@ -4155,22 +4040,6 @@ dependencies = [ "tonic 0.14.2", ] -[[package]] -name = "tonic-prost-build" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4a16cba4043dc3ff43fcb3f96b4c5c154c64cbd18ca8dce2ab2c6a451d058a2" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "prost-types 0.14.1", - "quote", - "syn", - "tempfile", - "tonic-build", -] - [[package]] name = "topiary-core" version = "0.6.1" diff --git a/Cargo.toml b/Cargo.toml index e585d1c75c..34271fe27c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,12 @@ indoc = "^2.0" nickel-lang-core = { version = "0.15.0", default-features = false, features = [ "format", ] } -object = {version = "0.37", default-features = false, features = ["archive", "elf", "std", "read_core"]} +object = { version = "0.37", default-features = false, features = [ + "archive", + "elf", + "std", + "read_core", +] } oci-client = { version = "0.15.0", features = [ "rustls-tls", ], default-features = false } diff --git a/crates/build-events-proto/Cargo.toml b/crates/build-events-proto/Cargo.toml index 8d09cbcd5d..aecdd1c9c3 100644 --- a/crates/build-events-proto/Cargo.toml +++ b/crates/build-events-proto/Cargo.toml @@ -8,9 +8,7 @@ publish.workspace = true [dependencies] build-events.workspace = true -prost.workspace = true -tonic.workspace = true -tonic-prost.workspace = true +tonic = "0.12.3" tokio.workspace = true thiserror.workspace = true tracing.workspace = true @@ -20,15 +18,13 @@ tokio-stream = { version = "0.1", features = ["sync"] } # Use workspace spongebob crate instead of direct BSR imports spongebob = { path = "../spongebob", optional = true } -# Need proto types for conversion when spongebob-subscriber feature is enabled -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251009015156-8bcd87750609.1", registry = "buf", optional = true } +# BSR proto dependencies for canonical spongebob.v1 types +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251009015156-8bcd87750609.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251009015156-8bcd87750609.1", registry = "buf" } [features] default = [] -spongebob-subscriber = ["spongebob", "minimal_spongebob_community_neoeinstein-prost"] - -[build-dependencies] -tonic-prost-build.workspace = true +spongebob-subscriber = ["spongebob"] [dev-dependencies] tempfile.workspace = true diff --git a/crates/build-events-proto/build.rs b/crates/build-events-proto/build.rs deleted file mode 100644 index 8172cf6ca3..0000000000 --- a/crates/build-events-proto/build.rs +++ /dev/null @@ -1,9 +0,0 @@ -fn main() -> Result<(), Box> { - // Compile protobuf files using tonic-prost-build - tonic_prost_build::compile_protos("proto/events.proto")?; - - // Re-run build script if proto files change - println!("cargo:rerun-if-changed=proto/events.proto"); - - Ok(()) -} diff --git a/crates/build-events-proto/proto/events.proto b/crates/build-events-proto/proto/events.proto deleted file mode 100644 index 2a2d271de0..0000000000 --- a/crates/build-events-proto/proto/events.proto +++ /dev/null @@ -1,91 +0,0 @@ -syntax = "proto3"; - -package build_events.v1; - -// BuildEvent is the top-level event wrapper that contains one of the specific event types -message BuildEvent { - oneof event { - BuildStarted build_started = 1; - BuildFinished build_finished = 2; - TargetStarted target_started = 3; - TargetCompleted target_completed = 4; - ActionStarted action_started = 5; - ActionCompleted action_completed = 6; - } -} - -// BuildStarted is emitted when a build invocation starts -message BuildStarted { - string invocation_id = 1; - int64 timestamp_millis = 3; - string working_directory = 4; - string command = 5; // Command string for display -} - -// BuildFinished is emitted when a build invocation completes -message BuildFinished { - string invocation_id = 1; - bool success = 2; - int64 timestamp_millis = 3; - optional string error_message = 4; -} - -// TargetStarted is emitted when a build target starts building -message TargetStarted { - string target_id = 1; - string label = 2; - TargetKind target_kind = 3; - int64 timestamp_millis = 4; -} - -// TargetCompleted is emitted when a build target completes -message TargetCompleted { - string target_id = 1; - string label = 2; - bool success = 3; - int64 timestamp_millis = 4; - optional string error_message = 5; - repeated string outputs = 6; - bool cache_hit = 7; -} - -// ActionStarted is emitted when a build action starts executing -message ActionStarted { - string action_id = 1; - string action_name = 2; - string target_id = 3; - int64 timestamp_millis = 4; -} - -// ActionCompleted is emitted when a build action completes -message ActionCompleted { - string action_id = 1; - bool success = 2; - int64 timestamp_millis = 3; - int32 exit_code = 4; - optional string stdout = 5; - optional string stderr = 6; -} - -// TargetKind represents the type of build target -enum TargetKind { - TARGET_KIND_UNSPECIFIED = 0; - TARGET_KIND_BINARY = 1; - TARGET_KIND_LIBRARY = 2; - TARGET_KIND_TEST = 3; -} - -// BuildEventService provides a streaming interface for build events -service BuildEventService { - // Subscribe to build events for a specific invocation or all invocations - rpc StreamEvents(StreamEventsRequest) returns (stream BuildEvent); -} - -// StreamEventsRequest configures the event stream subscription -message StreamEventsRequest { - // Optional filter to only receive events for a specific invocation - optional string invocation_id = 1; - - // Optional: start streaming from this timestamp (milliseconds since epoch) - optional int64 start_timestamp_millis = 2; -} diff --git a/crates/build-events-proto/src/convert.rs b/crates/build-events-proto/src/convert.rs index e41a888724..9df990c4fc 100644 --- a/crates/build-events-proto/src/convert.rs +++ b/crates/build-events-proto/src/convert.rs @@ -1,7 +1,25 @@ //! Type conversions between Rust and protobuf types //! -//! This module provides `From` implementations for converting between -//! the pure Rust `build_events` types and the generated protobuf types. +//! This module provides conversion functions between the pure Rust `build_events` +//! types and the BSR-generated canonical protobuf types from spongebob.v1. +//! +//! # Extension Trait +//! +//! For ergonomic conversion, use the [`ToProto`] trait which provides a `.to_proto()` method: +//! +//! ``` +//! use build_events::events::{BuildStarted, BuildEvent}; +//! use build_events_proto::convert::ToProto; +//! +//! let event = BuildEvent::BuildStarted(BuildStarted { +//! invocation_id: "test-123".to_string(), +//! command: "build".to_string(), +//! timestamp_millis: 123456789, +//! working_directory: "/tmp".to_string(), +//! }); +//! +//! let proto_event = event.to_proto("test-123"); +//! ``` use crate::proto; use build_events::events; @@ -10,111 +28,102 @@ use build_events::events; // Rust -> Proto conversions // ============================================================================ -impl From for proto::BuildEvent { - fn from(event: events::BuildEvent) -> Self { - let event = match event { - events::BuildEvent::BuildStarted(e) => { - proto::build_event::Event::BuildStarted(e.into()) - } - events::BuildEvent::BuildFinished(e) => { - proto::build_event::Event::BuildFinished(e.into()) - } - events::BuildEvent::TargetStarted(e) => { - proto::build_event::Event::TargetStarted(e.into()) - } - events::BuildEvent::TargetCompleted(e) => { - proto::build_event::Event::TargetCompleted(e.into()) - } - events::BuildEvent::ActionStarted(e) => { - proto::build_event::Event::ActionStarted(e.into()) - } - events::BuildEvent::ActionCompleted(e) => { - proto::build_event::Event::ActionCompleted(e.into()) - } - }; +/// Convert Rust BuildEvent to proto BuildEvent with invocation_id at wrapper level +/// +/// The canonical spongebob proto has `invocation_id` both at the BuildEvent wrapper level +/// and inside each event type. This function ensures the wrapper-level invocation_id is set. +pub fn to_proto_build_event(event: &events::BuildEvent, invocation_id: &str) -> proto::BuildEvent { + let proto_event = match event { + events::BuildEvent::BuildStarted(e) => { + proto::build_event::Event::BuildStarted(to_proto_build_started(e)) + } + events::BuildEvent::BuildFinished(e) => { + proto::build_event::Event::BuildFinished(to_proto_build_finished(e)) + } + events::BuildEvent::TargetStarted(e) => { + proto::build_event::Event::TargetStarted(to_proto_target_started(e)) + } + events::BuildEvent::TargetCompleted(e) => { + proto::build_event::Event::TargetCompleted(to_proto_target_completed(e)) + } + events::BuildEvent::ActionStarted(e) => { + proto::build_event::Event::ActionStarted(to_proto_action_started(e)) + } + events::BuildEvent::ActionCompleted(e) => { + proto::build_event::Event::ActionCompleted(to_proto_action_completed(e)) + } + }; - proto::BuildEvent { event: Some(event) } + proto::BuildEvent { + invocation_id: invocation_id.to_string(), + event: Some(proto_event), } } -impl From for proto::BuildStarted { - fn from(event: events::BuildStarted) -> Self { - proto::BuildStarted { - invocation_id: event.invocation_id, - command: event.command, - timestamp_millis: event.timestamp_millis, - working_directory: event.working_directory, - } +pub fn to_proto_build_started(event: &events::BuildStarted) -> proto::BuildStarted { + proto::BuildStarted { + invocation_id: event.invocation_id.clone(), + command: event.command.clone(), + timestamp_millis: event.timestamp_millis, + working_directory: event.working_directory.clone(), } } -impl From for proto::BuildFinished { - fn from(event: events::BuildFinished) -> Self { - proto::BuildFinished { - invocation_id: event.invocation_id, - success: event.success, - timestamp_millis: event.timestamp_millis, - error_message: event.error_message, - } +pub fn to_proto_build_finished(event: &events::BuildFinished) -> proto::BuildFinished { + proto::BuildFinished { + invocation_id: event.invocation_id.clone(), + success: event.success, + timestamp_millis: event.timestamp_millis, + error_message: event.error_message.clone(), } } -impl From for proto::TargetStarted { - fn from(event: events::TargetStarted) -> Self { - proto::TargetStarted { - target_id: event.target_id, - label: event.label, - target_kind: proto::TargetKind::from(event.target_kind) as i32, - timestamp_millis: event.timestamp_millis, - } +pub fn to_proto_target_started(event: &events::TargetStarted) -> proto::TargetStarted { + proto::TargetStarted { + target_id: event.target_id.clone(), + label: event.label.clone(), + target_kind: to_proto_target_kind(&event.target_kind) as i32, + timestamp_millis: event.timestamp_millis, } } -impl From for proto::TargetCompleted { - fn from(event: events::TargetCompleted) -> Self { - proto::TargetCompleted { - target_id: event.target_id, - label: event.label, - success: event.success, - timestamp_millis: event.timestamp_millis, - error_message: event.error_message, - outputs: event.outputs, - cache_hit: event.cache_hit, - } +pub fn to_proto_target_completed(event: &events::TargetCompleted) -> proto::TargetCompleted { + proto::TargetCompleted { + target_id: event.target_id.clone(), + label: event.label.clone(), + success: event.success, + timestamp_millis: event.timestamp_millis, + error_message: event.error_message.clone(), + outputs: event.outputs.clone(), + cache_hit: event.cache_hit, } } -impl From for proto::ActionStarted { - fn from(event: events::ActionStarted) -> Self { - proto::ActionStarted { - action_id: event.action_id, - action_name: event.action_name, - target_id: event.target_id, - timestamp_millis: event.timestamp_millis, - } +pub fn to_proto_action_started(event: &events::ActionStarted) -> proto::ActionStarted { + proto::ActionStarted { + action_id: event.action_id.clone(), + action_name: event.action_name.clone(), + target_id: event.target_id.clone(), + timestamp_millis: event.timestamp_millis, } } -impl From for proto::ActionCompleted { - fn from(event: events::ActionCompleted) -> Self { - proto::ActionCompleted { - action_id: event.action_id, - success: event.success, - timestamp_millis: event.timestamp_millis, - exit_code: event.exit_code, - stdout: event.stdout, - stderr: event.stderr, - } +pub fn to_proto_action_completed(event: &events::ActionCompleted) -> proto::ActionCompleted { + proto::ActionCompleted { + action_id: event.action_id.clone(), + success: event.success, + timestamp_millis: event.timestamp_millis, + exit_code: event.exit_code, + stdout: event.stdout.clone(), + stderr: event.stderr.clone(), } } -impl From for proto::TargetKind { - fn from(kind: events::TargetKind) -> Self { - match kind { - events::TargetKind::Binary => proto::TargetKind::Binary, - events::TargetKind::Library => proto::TargetKind::Library, - events::TargetKind::Test => proto::TargetKind::Test, - } +pub fn to_proto_target_kind(kind: &events::TargetKind) -> proto::TargetKind { + match kind { + events::TargetKind::Binary => proto::TargetKind::Binary, + events::TargetKind::Library => proto::TargetKind::Library, + events::TargetKind::Test => proto::TargetKind::Test, } } @@ -122,131 +131,155 @@ impl From for proto::TargetKind { // Proto -> Rust conversions // ============================================================================ -impl TryFrom for events::BuildEvent { - type Error = ConversionError; - - fn try_from(event: proto::BuildEvent) -> Result { - let event = event.event.ok_or(ConversionError::MissingField("event"))?; - - match event { - proto::build_event::Event::BuildStarted(e) => { - Ok(events::BuildEvent::BuildStarted(e.try_into()?)) - } - proto::build_event::Event::BuildFinished(e) => { - Ok(events::BuildEvent::BuildFinished(e.try_into()?)) - } - proto::build_event::Event::TargetStarted(e) => { - Ok(events::BuildEvent::TargetStarted(e.try_into()?)) - } - proto::build_event::Event::TargetCompleted(e) => { - Ok(events::BuildEvent::TargetCompleted(e.try_into()?)) - } - proto::build_event::Event::ActionStarted(e) => { - Ok(events::BuildEvent::ActionStarted(e.try_into()?)) - } - proto::build_event::Event::ActionCompleted(e) => { - Ok(events::BuildEvent::ActionCompleted(e.try_into()?)) - } - } +pub fn from_proto_build_event( + event: proto::BuildEvent, +) -> Result { + let event = event.event.ok_or(ConversionError::MissingField("event"))?; + + match event { + proto::build_event::Event::BuildStarted(e) => Ok(events::BuildEvent::BuildStarted( + from_proto_build_started(e)?, + )), + proto::build_event::Event::BuildFinished(e) => Ok(events::BuildEvent::BuildFinished( + from_proto_build_finished(e)?, + )), + proto::build_event::Event::TargetStarted(e) => Ok(events::BuildEvent::TargetStarted( + from_proto_target_started(e)?, + )), + proto::build_event::Event::TargetCompleted(e) => Ok(events::BuildEvent::TargetCompleted( + from_proto_target_completed(e)?, + )), + proto::build_event::Event::ActionStarted(e) => Ok(events::BuildEvent::ActionStarted( + from_proto_action_started(e)?, + )), + proto::build_event::Event::ActionCompleted(e) => Ok(events::BuildEvent::ActionCompleted( + from_proto_action_completed(e)?, + )), } } -impl TryFrom for events::BuildStarted { - type Error = ConversionError; - - fn try_from(event: proto::BuildStarted) -> Result { - Ok(events::BuildStarted { - invocation_id: event.invocation_id, - command: event.command, - timestamp_millis: event.timestamp_millis, - working_directory: event.working_directory, - }) - } +pub fn from_proto_build_started( + event: proto::BuildStarted, +) -> Result { + Ok(events::BuildStarted { + invocation_id: event.invocation_id, + command: event.command, + timestamp_millis: event.timestamp_millis, + working_directory: event.working_directory, + }) } -impl TryFrom for events::BuildFinished { - type Error = ConversionError; - - fn try_from(event: proto::BuildFinished) -> Result { - Ok(events::BuildFinished { - invocation_id: event.invocation_id, - success: event.success, - timestamp_millis: event.timestamp_millis, - error_message: event.error_message, - }) - } +pub fn from_proto_build_finished( + event: proto::BuildFinished, +) -> Result { + Ok(events::BuildFinished { + invocation_id: event.invocation_id, + success: event.success, + timestamp_millis: event.timestamp_millis, + error_message: event.error_message, + }) } -impl TryFrom for events::TargetStarted { - type Error = ConversionError; - - fn try_from(event: proto::TargetStarted) -> Result { - let target_kind = proto::TargetKind::try_from(event.target_kind) - .map_err(|_| ConversionError::InvalidEnum("target_kind", event.target_kind))?; - - Ok(events::TargetStarted { - target_id: event.target_id, - label: event.label, - target_kind: target_kind.try_into()?, - timestamp_millis: event.timestamp_millis, - }) - } +pub fn from_proto_target_started( + event: proto::TargetStarted, +) -> Result { + let target_kind = proto::TargetKind::try_from(event.target_kind) + .map_err(|_| ConversionError::InvalidEnum("target_kind", event.target_kind))?; + + Ok(events::TargetStarted { + target_id: event.target_id, + label: event.label, + target_kind: from_proto_target_kind(target_kind)?, + timestamp_millis: event.timestamp_millis, + }) } -impl TryFrom for events::TargetCompleted { - type Error = ConversionError; - - fn try_from(event: proto::TargetCompleted) -> Result { - Ok(events::TargetCompleted { - target_id: event.target_id, - label: event.label, - success: event.success, - timestamp_millis: event.timestamp_millis, - error_message: event.error_message, - outputs: event.outputs, - cache_hit: event.cache_hit, - }) - } +pub fn from_proto_target_completed( + event: proto::TargetCompleted, +) -> Result { + Ok(events::TargetCompleted { + target_id: event.target_id, + label: event.label, + success: event.success, + timestamp_millis: event.timestamp_millis, + error_message: event.error_message, + outputs: event.outputs, + cache_hit: event.cache_hit, + }) } -impl TryFrom for events::ActionStarted { - type Error = ConversionError; +pub fn from_proto_action_started( + event: proto::ActionStarted, +) -> Result { + Ok(events::ActionStarted { + action_id: event.action_id, + action_name: event.action_name, + target_id: event.target_id, + timestamp_millis: event.timestamp_millis, + }) +} - fn try_from(event: proto::ActionStarted) -> Result { - Ok(events::ActionStarted { - action_id: event.action_id, - action_name: event.action_name, - target_id: event.target_id, - timestamp_millis: event.timestamp_millis, - }) - } +pub fn from_proto_action_completed( + event: proto::ActionCompleted, +) -> Result { + Ok(events::ActionCompleted { + action_id: event.action_id, + success: event.success, + timestamp_millis: event.timestamp_millis, + exit_code: event.exit_code, + stdout: event.stdout, + stderr: event.stderr, + }) } -impl TryFrom for events::ActionCompleted { - type Error = ConversionError; - - fn try_from(event: proto::ActionCompleted) -> Result { - Ok(events::ActionCompleted { - action_id: event.action_id, - success: event.success, - timestamp_millis: event.timestamp_millis, - exit_code: event.exit_code, - stdout: event.stdout, - stderr: event.stderr, - }) +pub fn from_proto_target_kind( + kind: proto::TargetKind, +) -> Result { + match kind { + proto::TargetKind::Binary => Ok(events::TargetKind::Binary), + proto::TargetKind::Library => Ok(events::TargetKind::Library), + proto::TargetKind::Test => Ok(events::TargetKind::Test), + proto::TargetKind::Unspecified => Err(ConversionError::InvalidEnum("target_kind", 0)), } } -impl TryFrom for events::TargetKind { - type Error = ConversionError; +// ============================================================================ +// Extension Trait for Ergonomic Conversion +// ============================================================================ + +/// Extension trait providing ergonomic `.to_proto()` method for build events +/// +/// This trait provides a convenient way to convert Rust build events to protobuf +/// format with method syntax instead of calling standalone functions. +/// +/// # Example +/// +/// ``` +/// use build_events::events::{BuildStarted, BuildEvent, current_millis}; +/// use build_events_proto::convert::ToProto; +/// +/// let event = BuildEvent::BuildStarted(BuildStarted { +/// invocation_id: "test-123".to_string(), +/// command: "build".to_string(), +/// timestamp_millis: current_millis(), +/// working_directory: "/tmp".to_string(), +/// }); +/// +/// // Ergonomic method syntax +/// let proto_event = event.to_proto("test-123"); +/// ``` +pub trait ToProto { + /// Convert this event to a protobuf BuildEvent with the specified invocation_id + /// + /// The invocation_id is required because the canonical spongebob proto has + /// `invocation_id` at the BuildEvent wrapper level, separate from the inner + /// event types. + fn to_proto(&self, invocation_id: &str) -> proto::BuildEvent; +} - fn try_from(kind: proto::TargetKind) -> Result { - match kind { - proto::TargetKind::Binary => Ok(events::TargetKind::Binary), - proto::TargetKind::Library => Ok(events::TargetKind::Library), - proto::TargetKind::Test => Ok(events::TargetKind::Test), - proto::TargetKind::Unspecified => Err(ConversionError::InvalidEnum("target_kind", 0)), - } +impl ToProto for events::BuildEvent { + fn to_proto(&self, invocation_id: &str) -> proto::BuildEvent { + to_proto_build_event(self, invocation_id) } } @@ -277,8 +310,8 @@ mod tests { working_directory: "/tmp".to_string(), }; - let proto_event: proto::BuildStarted = rust_event.clone().into(); - let converted_back: events::BuildStarted = proto_event.try_into().unwrap(); + let proto_event = to_proto_build_started(&rust_event); + let converted_back = from_proto_build_started(proto_event).unwrap(); assert_eq!(rust_event, converted_back); } @@ -292,8 +325,8 @@ mod tests { working_directory: "/home/user".to_string(), }); - let proto_event: proto::BuildEvent = rust_event.clone().into(); - let converted_back: events::BuildEvent = proto_event.try_into().unwrap(); + let proto_event = to_proto_build_event(&rust_event, "test-456"); + let converted_back = from_proto_build_event(proto_event).unwrap(); assert_eq!(rust_event, converted_back); } @@ -301,8 +334,8 @@ mod tests { #[test] fn test_target_kind_conversion() { let rust_kind = events::TargetKind::Binary; - let proto_kind: proto::TargetKind = rust_kind.clone().into(); - let converted_back: events::TargetKind = proto_kind.try_into().unwrap(); + let proto_kind = to_proto_target_kind(&rust_kind); + let converted_back = from_proto_target_kind(proto_kind).unwrap(); assert_eq!(rust_kind, converted_back); } @@ -316,8 +349,8 @@ mod tests { error_message: Some("build failed".to_string()), }; - let proto_event: proto::BuildFinished = rust_event.clone().into(); - let converted_back: events::BuildFinished = proto_event.try_into().unwrap(); + let proto_event = to_proto_build_finished(&rust_event); + let converted_back = from_proto_build_finished(proto_event).unwrap(); assert_eq!(rust_event, converted_back); @@ -329,8 +362,8 @@ mod tests { error_message: None, }; - let proto_event_none: proto::BuildFinished = rust_event_none.clone().into(); - let converted_back_none: events::BuildFinished = proto_event_none.try_into().unwrap(); + let proto_event_none = to_proto_build_finished(&rust_event_none); + let converted_back_none = from_proto_build_finished(proto_event_none).unwrap(); assert_eq!(rust_event_none, converted_back_none); } diff --git a/crates/build-events-proto/src/lib.rs b/crates/build-events-proto/src/lib.rs index dfbcca99b4..bc13418386 100644 --- a/crates/build-events-proto/src/lib.rs +++ b/crates/build-events-proto/src/lib.rs @@ -113,22 +113,26 @@ //! assert_eq!(rust_event, converted_back); //! ``` -// Include generated protobuf code +// Use BSR-generated protobuf code from canonical spongebob.v1 package pub mod proto { - tonic::include_proto!("build_events.v1"); + // Re-export message types from prost + pub use minimal_spongebob_community_neoeinstein_prost::spongebob::v1::*; + + // Re-export service types from tonic + pub use minimal_spongebob_community_neoeinstein_tonic::spongebob::v1::tonic::{ + build_event_service_client, build_event_service_server, + }; } pub mod convert; pub mod service; pub mod subscriber; -#[cfg(feature = "spongebob-subscriber")] -pub mod spongebob_convert; #[cfg(feature = "spongebob-subscriber")] pub mod spongebob_subscriber_v2; // Re-export main types for convenience -pub use convert::ConversionError; +pub use convert::{ConversionError, ToProto, to_proto_build_event}; pub use service::BuildEventServiceImpl; pub use subscriber::GrpcStreamSubscriber; diff --git a/crates/build-events-proto/src/service.rs b/crates/build-events-proto/src/service.rs index e9791e6f75..05b508759a 100644 --- a/crates/build-events-proto/src/service.rs +++ b/crates/build-events-proto/src/service.rs @@ -6,7 +6,6 @@ use tokio::sync::broadcast; use tokio_stream::wrappers::BroadcastStream; use tokio_stream::{Stream, StreamExt}; -use tonic::{Request, Response, Status}; use tracing::{info, warn}; use crate::proto; @@ -102,13 +101,14 @@ impl BuildEventServiceImpl { #[tonic::async_trait] impl BuildEventService for BuildEventServiceImpl { - type StreamEventsStream = - std::pin::Pin> + Send + 'static>>; + type StreamBuildEventsStream = std::pin::Pin< + Box> + Send + 'static>, + >; - async fn stream_events( + async fn stream_build_events( &self, - request: Request, - ) -> Result, Status> { + request: tonic::Request, + ) -> Result, tonic::Status> { let req = request.into_inner(); info!( @@ -172,12 +172,32 @@ impl BuildEventService for BuildEventServiceImpl { Err(e) => { warn!("Broadcast stream error: {}", e); // Convert to gRPC status - Some(Err(Status::internal(format!("Stream error: {}", e)))) + Some(Err(tonic::Status::internal(format!("Stream error: {}", e)))) } } }); - Ok(Response::new(Box::pin(filtered_stream))) + Ok(tonic::Response::new(Box::pin(filtered_stream))) + } + + async fn publish_build_event( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let req = request.into_inner(); + + // Extract the event from the request + let event = req + .event + .ok_or_else(|| tonic::Status::invalid_argument("Missing event in request"))?; + + // Publish to broadcast channel + self.publish(event); + + // Return success response with empty name (will be set by server) + Ok(tonic::Response::new(proto::PublishBuildEventResponse { + name: String::new(), + })) } } @@ -194,7 +214,10 @@ mod tests { #[test] fn test_publish_no_subscribers() { let service = BuildEventServiceImpl::new(100); - let event = proto::BuildEvent { event: None }; + let event = proto::BuildEvent { + invocation_id: String::new(), + event: None, + }; // Should not panic service.publish(event); } diff --git a/crates/build-events-proto/src/spongebob_convert.rs b/crates/build-events-proto/src/spongebob_convert.rs deleted file mode 100644 index 5f58d3ecc8..0000000000 --- a/crates/build-events-proto/src/spongebob_convert.rs +++ /dev/null @@ -1,165 +0,0 @@ -//! Conversion between build-events Rust types and spongebob proto types -//! -//! This module is only available when the `spongebob-subscriber` feature is enabled. - -use build_events::events::{ - ActionCompleted, ActionStarted, BuildEvent, BuildFinished, BuildStarted, TargetCompleted, - TargetKind, TargetStarted, -}; - -// Re-export the proto types from the spongebob crate -pub use minimal_spongebob_community_neoeinstein_prost::spongebob::v1 as pb; - -/// Convert Rust BuildEvent to spongebob proto BuildEvent -/// Note: This function requires an invocation_id to be passed separately since the -/// Rust BuildEvent enum doesn't carry it at the wrapper level -pub fn to_proto_build_event(event: &BuildEvent, invocation_id: &str) -> pb::BuildEvent { - match event { - BuildEvent::BuildStarted(e) => pb::BuildEvent { - invocation_id: invocation_id.to_string(), - event: Some(pb::build_event::Event::BuildStarted( - to_proto_build_started(e), - )), - }, - BuildEvent::BuildFinished(e) => pb::BuildEvent { - invocation_id: invocation_id.to_string(), - event: Some(pb::build_event::Event::BuildFinished( - to_proto_build_finished(e), - )), - }, - BuildEvent::TargetStarted(e) => pb::BuildEvent { - invocation_id: invocation_id.to_string(), - event: Some(pb::build_event::Event::TargetStarted( - to_proto_target_started(e), - )), - }, - BuildEvent::TargetCompleted(e) => pb::BuildEvent { - invocation_id: invocation_id.to_string(), - event: Some(pb::build_event::Event::TargetCompleted( - to_proto_target_completed(e), - )), - }, - BuildEvent::ActionStarted(e) => pb::BuildEvent { - invocation_id: invocation_id.to_string(), - event: Some(pb::build_event::Event::ActionStarted( - to_proto_action_started(e), - )), - }, - BuildEvent::ActionCompleted(e) => pb::BuildEvent { - invocation_id: invocation_id.to_string(), - event: Some(pb::build_event::Event::ActionCompleted( - to_proto_action_completed(e), - )), - }, - } -} - -fn to_proto_build_started(e: &BuildStarted) -> pb::BuildStarted { - pb::BuildStarted { - invocation_id: e.invocation_id.clone(), - command: e.command.clone(), - timestamp_millis: e.timestamp_millis, - working_directory: e.working_directory.clone(), - } -} - -fn to_proto_build_finished(e: &BuildFinished) -> pb::BuildFinished { - pb::BuildFinished { - invocation_id: e.invocation_id.clone(), - success: e.success, - timestamp_millis: e.timestamp_millis, - error_message: e.error_message.clone(), - } -} - -fn to_proto_target_started(e: &TargetStarted) -> pb::TargetStarted { - pb::TargetStarted { - target_id: e.target_id.clone(), - label: e.label.clone(), - target_kind: to_proto_target_kind(&e.target_kind) as i32, - timestamp_millis: e.timestamp_millis, - } -} - -fn to_proto_target_completed(e: &TargetCompleted) -> pb::TargetCompleted { - pb::TargetCompleted { - target_id: e.target_id.clone(), - label: e.label.clone(), - success: e.success, - timestamp_millis: e.timestamp_millis, - error_message: e.error_message.clone(), - outputs: e.outputs.clone(), - cache_hit: e.cache_hit, - } -} - -fn to_proto_action_started(e: &ActionStarted) -> pb::ActionStarted { - pb::ActionStarted { - action_id: e.action_id.clone(), - action_name: e.action_name.clone(), - target_id: e.target_id.clone(), - timestamp_millis: e.timestamp_millis, - } -} - -fn to_proto_action_completed(e: &ActionCompleted) -> pb::ActionCompleted { - pb::ActionCompleted { - action_id: e.action_id.clone(), - success: e.success, - timestamp_millis: e.timestamp_millis, - exit_code: e.exit_code, - stdout: e.stdout.clone(), - stderr: e.stderr.clone(), - } -} - -fn to_proto_target_kind(kind: &TargetKind) -> pb::TargetKind { - match kind { - TargetKind::Binary => pb::TargetKind::Binary, - TargetKind::Library => pb::TargetKind::Library, - TargetKind::Test => pb::TargetKind::Test, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use build_events::events::current_millis; - - #[test] - fn test_build_started_conversion() { - let rust_event = BuildEvent::BuildStarted(BuildStarted { - invocation_id: "test-123".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - }); - - let proto_event: pb::BuildEvent = to_proto_build_event(&rust_event, "test-123"); - assert_eq!(proto_event.invocation_id, "test-123"); - assert!(proto_event.event.is_some()); - match proto_event.event.unwrap() { - pb::build_event::Event::BuildStarted(e) => { - assert_eq!(e.invocation_id, "test-123"); - assert_eq!(e.command, "build"); - } - _ => panic!("Wrong event type"), - } - } - - #[test] - fn test_target_kind_conversion() { - assert_eq!( - to_proto_target_kind(&TargetKind::Binary) as i32, - pb::TargetKind::Binary as i32 - ); - assert_eq!( - to_proto_target_kind(&TargetKind::Library) as i32, - pb::TargetKind::Library as i32 - ); - assert_eq!( - to_proto_target_kind(&TargetKind::Test) as i32, - pb::TargetKind::Test as i32 - ); - } -} diff --git a/crates/build-events-proto/src/spongebob_subscriber_v2.rs b/crates/build-events-proto/src/spongebob_subscriber_v2.rs index cd1ae1b924..3056017606 100644 --- a/crates/build-events-proto/src/spongebob_subscriber_v2.rs +++ b/crates/build-events-proto/src/spongebob_subscriber_v2.rs @@ -8,7 +8,7 @@ use build_events::{BuildEvent, BuildEventSubscriber, SubscriberError}; use tracing::warn; #[cfg(feature = "spongebob-subscriber")] -use crate::spongebob_convert::to_proto_build_event; +use crate::convert::ToProto; /// Subscriber that publishes build events to Spongebob service /// @@ -72,7 +72,7 @@ impl BuildEventSubscriber for SpongeBobSubscriberV2 { let mut invocation = self.invocation.lock().await; // Convert Rust event to proto (need invocation_id from the invocation) - let proto_event = to_proto_build_event(event, invocation.invocation_id()); + let proto_event = event.to_proto(invocation.invocation_id()); // Publish event match invocation.publish_build_event(proto_event).await { diff --git a/crates/build-events-proto/src/subscriber.rs b/crates/build-events-proto/src/subscriber.rs index 417d0f8ee0..efc081071d 100644 --- a/crates/build-events-proto/src/subscriber.rs +++ b/crates/build-events-proto/src/subscriber.rs @@ -89,8 +89,19 @@ impl GrpcStreamSubscriber { #[async_trait] impl BuildEventSubscriber for GrpcStreamSubscriber { async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { - // Convert Rust event to proto - let proto_event: proto::BuildEvent = event.clone().into(); + // Extract invocation_id from the event (all variants have it) + let invocation_id = match event { + BuildEvent::BuildStarted(e) => &e.invocation_id, + BuildEvent::BuildFinished(e) => &e.invocation_id, + BuildEvent::TargetStarted(_) => "", // Targets don't have invocation_id in inner fields + BuildEvent::TargetCompleted(_) => "", + BuildEvent::ActionStarted(_) => "", + BuildEvent::ActionCompleted(_) => "", + }; + + // Convert Rust event to proto using ergonomic extension trait + use crate::convert::ToProto; + let proto_event = event.to_proto(invocation_id); // Broadcast to all subscribers // If there are no subscribers, this is a no-op From c8f91b16eb7d453ecc09badbdc930b383501f99d Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Thu, 9 Oct 2025 13:53:03 -0700 Subject: [PATCH 05/39] update --- CLAUDE.md | 17 ++++- crates/build-sandbox/src/executor.rs | 65 +++++++++++++++++-- crates/build-sandbox/src/executor_stub.rs | 2 + crates/build-sandbox/src/lib.rs | 6 +- crates/minimal/src/cmd_build.rs | 11 +++- .../minimal/src/cmd_unsafe_patched_build.rs | 5 ++ crates/minimal/src/run.rs | 4 ++ crates/spongebob/src/lib.rs | 10 ++- deny.toml | 13 ++++ 9 files changed, 123 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 364dfaa97e..308206745d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -93,11 +93,18 @@ Package Spec (build.ncl) → SpecReader → DepGraph → ExecPlan → Build Reso **Sandbox Isolation Strategy**: - **Linux**: User namespaces + mount namespaces for filesystem isolation -- **macOS**: `sandbox-exec` with dynamically generated policies +- **macOS**: `sandbox-exec` with dynamically generated policies - **Dependency mounting**: Build dependencies mounted read-only at specific paths - **Input copying**: Source files and local inputs copied to TMPDIR - **Output staging**: Builds write to `OUTPUT_DIR`, enumerated outputs copied to cache +**SpongeBob Integration**: +- **Build events**: Executor emits TargetStarted and TargetCompleted events for observability +- **Target identifiers**: Package names (e.g., "tar", "glibc") used as target IDs +- **File uploads**: stdout/stderr uploaded to SpongeBob after build execution +- **Event flow**: TargetStarted → Build executes → Files uploaded → TargetCompleted +- **Error handling**: Events published even on build failure for complete observability + ### Package Ecosystem **Current Package Count**: 41 packages including: @@ -175,6 +182,14 @@ gcc, - Linux uses direct syscalls via `nix` crate (no external dependencies) - Error handling includes platform-specific sandbox violations +**Build Observability (SpongeBob Integration)**: +- **build-sandbox executor** emits build events automatically when a SpongeBobInvocation is provided +- **TargetStarted event**: Published before build execution begins, creates target in database +- **TargetCompleted event**: Published after build finishes (success or failure) +- **File uploads**: stdout/stderr automatically uploaded to SpongeBob for each target +- **Target naming**: Uses package name as semantic target identifier (e.g., "tar" not a random UUID) +- **Requirement**: Targets must exist (via TargetStarted) before file uploads succeed + ## Working with Package Specs Package specifications are written in Nickel in `packages//build.ncl`: diff --git a/crates/build-sandbox/src/executor.rs b/crates/build-sandbox/src/executor.rs index e6543e9c77..da0a5ed1d4 100644 --- a/crates/build-sandbox/src/executor.rs +++ b/crates/build-sandbox/src/executor.rs @@ -1,4 +1,7 @@ use hakoniwa::Container; +use minimal_spongebob_community_neoeinstein_prost::spongebob::v1::{ + build_event, BuildEvent, TargetCompleted, TargetKind, TargetStarted, +}; use std::fs; use std::os::unix::fs::symlink; use std::path::{Path, PathBuf}; @@ -58,6 +61,7 @@ impl BuildExecutor { &self, config: &BuildConfig, spongebob_invocation: &mut Option, + target_id: &str, ) -> Result<(i32, Option)> { info!( "Linking {} inputs to build environment", @@ -68,8 +72,52 @@ impl BuildExecutor { self.hardlink_to_tmpdir(input)?; } - self.execute_in_container(config, spongebob_invocation) - .await?; + // Emit TargetStarted event + if let Some(invocation) = spongebob_invocation { + let target_started = BuildEvent { + invocation_id: invocation.invocation_id().to_string(), + event: Some(build_event::Event::TargetStarted(TargetStarted { + target_id: target_id.to_string(), + label: config.name.clone(), + target_kind: TargetKind::Binary as i32, + timestamp_millis: current_millis(), + })), + }; + if let Err(e) = invocation.publish_build_event(target_started).await { + warn!("Failed to publish TargetStarted event: {}", e); + } + } + + let execute_result = self + .execute_in_container(config, spongebob_invocation, target_id) + .await; + + // Emit TargetCompleted event + if let Some(invocation) = spongebob_invocation { + let (success, error_message) = match &execute_result { + Ok(_) => (true, None), + Err(e) => (false, Some(e.to_string())), + }; + + let target_completed = BuildEvent { + invocation_id: invocation.invocation_id().to_string(), + event: Some(build_event::Event::TargetCompleted(TargetCompleted { + target_id: target_id.to_string(), + label: config.name.clone(), + success, + timestamp_millis: current_millis(), + error_message, + outputs: vec![], // Outputs are collected later + cache_hit: false, // Cache hit detection happens earlier + })), + }; + if let Err(e) = invocation.publish_build_event(target_completed).await { + warn!("Failed to publish TargetCompleted event: {}", e); + } + } + + // Return the original result + execute_result?; let spongebob_url = spongebob_invocation .as_ref() @@ -126,6 +174,7 @@ impl BuildExecutor { &self, config: &BuildConfig, spongebob_invocation: &mut Option, + target_id: &str, ) -> Result<()> { let rootfs = self.prepare_rootfs(config)?; let sandbox_mount_point = "/build"; @@ -200,7 +249,7 @@ impl BuildExecutor { if let Some(invocation) = spongebob_invocation { if let Err(e) = invocation - .upload_file("stdout", output.stdout.to_vec()) + .upload_file(target_id, "stdout", output.stdout.to_vec()) .await { warn!( @@ -210,7 +259,7 @@ impl BuildExecutor { } if let Err(e) = invocation - .upload_file("stderr", output.stderr.to_vec()) + .upload_file(target_id, "stderr", output.stderr.to_vec()) .await { warn!( @@ -336,3 +385,11 @@ fn hardlink_dir_contents(src: &Path, dst: &Path) -> Result<()> { Ok(()) } + +/// Helper function to get current timestamp in milliseconds +fn current_millis() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("System time before UNIX epoch") + .as_millis() as i64 +} diff --git a/crates/build-sandbox/src/executor_stub.rs b/crates/build-sandbox/src/executor_stub.rs index a9676ad939..234416bdee 100644 --- a/crates/build-sandbox/src/executor_stub.rs +++ b/crates/build-sandbox/src/executor_stub.rs @@ -17,6 +17,7 @@ impl BuildExecutor { &self, _: &BuildConfig, _: &mut Option, + _: &str, ) -> Result<(i32, Option)> { todo!() } @@ -37,6 +38,7 @@ impl BuildExecutor { &self, _: &BuildConfig, _: &mut Option, + _: &str, ) -> Result<()> { todo!() } diff --git a/crates/build-sandbox/src/lib.rs b/crates/build-sandbox/src/lib.rs index 524b777e1d..c7eff51dfb 100644 --- a/crates/build-sandbox/src/lib.rs +++ b/crates/build-sandbox/src/lib.rs @@ -24,15 +24,19 @@ pub struct BuildResult { /// * `config` - Build configuration including inputs, outputs, and build script /// * `cache_dest_dir` - Final destination in cache where successful build outputs are stored /// * `sandbox_base_dir` - Base directory for creating temporary build sandboxes +/// * `target_id` - Unique identifier for the target being built #[tracing::instrument(skip_all, fields(name = config.name, indicatif.pb_show))] pub async fn run_build( config: &BuildConfig, cache_dest_dir: &std::path::Path, spongebob_invocation: &mut Option, sandbox_base_dir: std::path::PathBuf, + target_id: &str, ) -> Result { let executor = BuildExecutor::new(sandbox_base_dir, config.name.clone())?; - let (exit_code, spongebob_url) = executor.execute(config, spongebob_invocation).await?; + let (exit_code, spongebob_url) = executor + .execute(config, spongebob_invocation, target_id) + .await?; let outputs = OutputValidator::validate_and_collect( config, diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index e646e5a56f..4469bfa2a4 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -250,10 +250,17 @@ async fn log_build_results_to_spongebob( graph.top_levels.len() ); - // Upload build summary + // Use a semantic target ID for the build command summary + let build_command_target_id = "build-summary".to_string(); + + // Upload build summary to the synthetic target info!("Uploading build summary to SpongeBob invocation"); spongebob_invocation - .upload_file("build-summary.txt", build_log.into_bytes()) + .upload_file( + &build_command_target_id, + "build-summary.txt", + build_log.into_bytes(), + ) .await .map_err(|e| anyhow::anyhow!("Failed to upload build summary to SpongeBob: {}", e))?; diff --git a/crates/minimal/src/cmd_unsafe_patched_build.rs b/crates/minimal/src/cmd_unsafe_patched_build.rs index d795d94a76..ca70b527f1 100644 --- a/crates/minimal/src/cmd_unsafe_patched_build.rs +++ b/crates/minimal/src/cmd_unsafe_patched_build.rs @@ -91,11 +91,16 @@ pub async fn cmd_unsafe_patched_build( .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; let mut spongebob_invocation = Some(spongebob_client.create_invocation()); + + // Use package name as target ID for semantic meaning + let target_id = build.name.clone(); + run_build( &config, out_dir.path(), &mut spongebob_invocation, output_base.clone(), + &target_id, ) .await .map_err(|e| anyhow::anyhow!("Failed to build {}: {}", build.name, e))?; diff --git a/crates/minimal/src/run.rs b/crates/minimal/src/run.rs index 07c2c9e45c..f9fb46eb51 100644 --- a/crates/minimal/src/run.rs +++ b/crates/minimal/src/run.rs @@ -500,12 +500,16 @@ impl<'a> Run<'a> { let out_dir = self.cache.write_dir(&bsh).unwrap(); + // Use package name as target ID for semantic meaning + let target_id = build.name.clone(); + info!("Building package: {}", build.name); let build_result = run_build( &config, out_dir.path(), spongebob_invocation, self.output_base.clone(), + &target_id, ) .await .map_err(|e| anyhow::anyhow!("Failed to build {}: {}", build.name, e))?; diff --git a/crates/spongebob/src/lib.rs b/crates/spongebob/src/lib.rs index 56c146bf88..78fb150c28 100644 --- a/crates/spongebob/src/lib.rs +++ b/crates/spongebob/src/lib.rs @@ -53,10 +53,16 @@ impl SpongeBobInvocation { } } + /// Upload a file to a specific target within this invocation #[tracing::instrument(skip(self, contents))] - pub async fn upload_file(&mut self, file_name: &str, contents: Vec) -> Result<()> { + pub async fn upload_file( + &mut self, + target_id: &str, + file_name: &str, + contents: Vec, + ) -> Result<()> { let file_id = Uuid::new_v4().to_string(); - let parent = format!("invocations/{}", self.invocation_id); + let parent = format!("invocations/{}/targets/{}", self.invocation_id, target_id); let request = CreateFileRequest { parent, file_id, diff --git a/deny.toml b/deny.toml index 24e494eca2..a565347d37 100644 --- a/deny.toml +++ b/deny.toml @@ -68,6 +68,19 @@ skip = [ # tokio -> mio { name = "wasi", version = "=0.11.1+wasi-snapshot-preview1" }, + + { name = "tower", version = "0.4.13" }, + { name = "tonic", version = "0.12.3" }, + { name = "socket2", version = "0.5.10" }, + { name = "rand_core", version = "0.6.4" }, + { name = "rand_chacha", version = "0.3.1" }, + { name = "rand", version = "0.8.5" }, + { name = "prost-types", version = "0.13.5" }, + { name = "prost-derive", version = "0.13.5" }, + { name = "prost", version = "0.13.5" }, + { name = "indexmap", version = "1.9.3" }, + + ] skip-tree = [ From 431574ab38705b55817424b9f6fb750cb2c02752 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Fri, 10 Oct 2025 12:47:09 -0700 Subject: [PATCH 06/39] downgrade spongbob warning to debug --- crates/minimal/src/cmd_build.rs | 18 +++++------------- crates/minimal/src/cmd_nw_update.rs | 1 - crates/minimal/src/cmd_oci_image.rs | 1 - crates/minimal/src/cmd_run.rs | 1 - 4 files changed, 5 insertions(+), 16 deletions(-) diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index 50c0a0697d..0dba4dafb8 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -16,7 +16,7 @@ pub async fn cmd_build(args: BuildArgs, globals: &GlobalArgs) -> Result<(), Erro let graph = args.packages.graph(globals)?; let cache = globals.cache().map_err(anyhow::Error::from)?; - cmd_build_impl(&graph, globals, cache, globals.num_parallel_builds, false).await?; + cmd_build_impl(&graph, globals, cache, globals.num_parallel_builds).await?; Ok(()) } @@ -26,7 +26,6 @@ pub async fn cmd_build_impl( globals: &GlobalArgs, cache: Cache, num_parallel_builds: usize, - quiet: bool, ) -> anyhow::Result<()> { // Create SpongeBob invocation for this build command let mut spongebob_client = spongebob::SpongeBob::new() @@ -64,9 +63,7 @@ pub async fn cmd_build_impl( let mut spongebob_invocation = match spongebob_client.create_invocation(&command_info).await { Ok(inv) => Some(inv), Err(e) => { - if !quiet { - tracing::warn!("Failed to create SpongeBob invocation: {}", e); - } + tracing::debug!("Failed to create SpongeBob invocation: {}", e); None } }; @@ -165,7 +162,7 @@ pub async fn cmd_build_impl( // Log build results to SpongeBob if let Some(ref mut invocation) = spongebob_invocation { - log_build_results_to_spongebob(invocation, graph, true, quiet).await?; + log_build_results_to_spongebob(invocation, graph, true).await?; } let spongebob_url = spongebob_invocation @@ -173,9 +170,7 @@ pub async fn cmd_build_impl( .map(|inv| inv.url().to_string()); // Display build summary - if !quiet { - display_build_summary(graph, &cache, globals, &run, spongebob_url.as_deref()); - } + display_build_summary(graph, &cache, globals, &run, spongebob_url.as_deref()); Ok(()) } @@ -185,7 +180,6 @@ async fn log_build_results_to_spongebob( spongebob_invocation: &mut spongebob::SpongeBobInvocation, graph: &DepGraph, success: bool, - quiet: bool, ) -> anyhow::Result<()> { let packages_list = graph .top_levels @@ -214,9 +208,7 @@ async fn log_build_results_to_spongebob( ); // Upload build summary - if !quiet { - info!("Uploading build summary to SpongeBob invocation"); - } + info!("Uploading build summary to SpongeBob invocation"); spongebob_invocation .upload_file("build-summary.txt", build_log.into_bytes()) .await diff --git a/crates/minimal/src/cmd_nw_update.rs b/crates/minimal/src/cmd_nw_update.rs index 0aef41c8df..b263b1bd46 100644 --- a/crates/minimal/src/cmd_nw_update.rs +++ b/crates/minimal/src/cmd_nw_update.rs @@ -22,7 +22,6 @@ pub async fn cmd_new_world_update(args: NWUpdateArgs, globals: &GlobalArgs) -> R globals, cache.clone(), globals.num_parallel_builds, - true, ) .await?; diff --git a/crates/minimal/src/cmd_oci_image.rs b/crates/minimal/src/cmd_oci_image.rs index 564ce0f5ab..39842877d3 100644 --- a/crates/minimal/src/cmd_oci_image.rs +++ b/crates/minimal/src/cmd_oci_image.rs @@ -51,7 +51,6 @@ pub async fn cmd_oci_image(args: OciImageArgs, globals: &GlobalArgs) -> Result<( globals, cache.clone(), globals.num_parallel_builds, - true, ) .await?; diff --git a/crates/minimal/src/cmd_run.rs b/crates/minimal/src/cmd_run.rs index ece0088bc9..c50c6f982b 100644 --- a/crates/minimal/src/cmd_run.rs +++ b/crates/minimal/src/cmd_run.rs @@ -28,7 +28,6 @@ pub async fn cmd_run(args: RunArgs, globals: &GlobalArgs) -> Result<(), Error> { globals, cache.clone(), globals.num_parallel_builds, - true, ) .await?; From 05f5eb903be43c4694a512b73d74eeb8f649d87b Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Fri, 10 Oct 2025 13:05:06 -0700 Subject: [PATCH 07/39] fix linux build --- Cargo.lock | 1 + crates/build-sandbox/Cargo.toml | 3 +++ crates/minimal/src/cmd_patched_build.rs | 13 +------------ 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ada7eef8a8..3f1ce6592f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -404,6 +404,7 @@ dependencies = [ "common", "globset", "hakoniwa", + "minimal_spongebob_community_neoeinstein-prost", "nickel-lang-core", "serde", "spongebob", diff --git a/crates/build-sandbox/Cargo.toml b/crates/build-sandbox/Cargo.toml index 6545870f08..36f332befb 100644 --- a/crates/build-sandbox/Cargo.toml +++ b/crates/build-sandbox/Cargo.toml @@ -17,3 +17,6 @@ common.workspace = true [target.'cfg(target_os = "linux")'.dependencies] hakoniwa.workspace = true + +# TODO, get this out of the build-sandbox crate +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251009015156-8bcd87750609.1", registry = "buf" } diff --git a/crates/minimal/src/cmd_patched_build.rs b/crates/minimal/src/cmd_patched_build.rs index c9e913ed11..e71e6188a8 100644 --- a/crates/minimal/src/cmd_patched_build.rs +++ b/crates/minimal/src/cmd_patched_build.rs @@ -89,21 +89,10 @@ pub async fn cmd_patched_build(args: PatchedBuildArgs, globals: &GlobalArgs) -> .await .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; - let mut spongebob_invocation = Some(spongebob_client.create_invocation()); // Use package name as target ID for semantic meaning let target_id = build.name.clone(); - let command_info = format!("unsafe-patched-build {}", build.name); - let mut spongebob_client = spongebob::SpongeBob::new() - .await - .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; - let mut spongebob_invocation = match spongebob_client.create_invocation(&command_info).await { - Ok(inv) => Some(inv), - Err(e) => { - tracing::warn!("Failed to create SpongeBob invocation: {}", e); - None - } - }; + let mut spongebob_invocation = Some(spongebob_client.create_invocation()); run_build( &config, From 511c41602e70096008083d8f5e2ddbfe009eeb87 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Fri, 10 Oct 2025 13:18:13 -0700 Subject: [PATCH 08/39] stop using UUIDs again --- crates/spongebob/src/lib.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/spongebob/src/lib.rs b/crates/spongebob/src/lib.rs index 78fb150c28..90eff1a7ca 100644 --- a/crates/spongebob/src/lib.rs +++ b/crates/spongebob/src/lib.rs @@ -61,11 +61,10 @@ impl SpongeBobInvocation { file_name: &str, contents: Vec, ) -> Result<()> { - let file_id = Uuid::new_v4().to_string(); let parent = format!("invocations/{}/targets/{}", self.invocation_id, target_id); let request = CreateFileRequest { parent, - file_id, + file_id: file_name.to_string(), file: Some(File { name: file_name.to_string(), contents: contents.into(), From f4c221a1def67a65a30adb7551f054814576e120 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Tue, 14 Oct 2025 09:42:47 -0700 Subject: [PATCH 09/39] build metadata event --- Cargo.lock | 28 ++++++----- crates/build-events-proto/Cargo.toml | 4 +- crates/build-events-proto/src/convert.rs | 20 ++++++++ crates/build-events-proto/src/service.rs | 1 + crates/build-events-proto/src/subscriber.rs | 1 + crates/build-events/src/events.rs | 11 +++++ crates/build-events/src/subscribers/logger.rs | 7 +++ crates/build-sandbox/Cargo.toml | 2 +- crates/minimal/Cargo.toml | 1 + crates/minimal/src/cmd_build.rs | 48 ++++++++++++++++++- crates/minimal/src/main.rs | 4 ++ crates/spongebob/Cargo.toml | 4 +- 12 files changed, 114 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f1ce6592f..7d4edd3049 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -864,7 +864,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -938,7 +938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1458,6 +1458,12 @@ dependencies = [ "foldhash", ] +[[package]] +name = "hashbrown" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" + [[package]] name = "hashlink" version = "0.10.0" @@ -1808,7 +1814,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown 0.15.5", + "hashbrown 0.16.0", "serde", "serde_core", ] @@ -2222,9 +2228,9 @@ dependencies = [ [[package]] name = "minimal_spongebob_community_neoeinstein-prost" -version = "0.4.0-20251009015156-8bcd87750609.1" +version = "0.4.0-20251010205210-42ae8fd1517e.1" source = "sparse+https://buf.build/gen/cargo/" -checksum = "699b4089a62d6370864b932906ff4ba10eec4485cf9ea0ef3d868dd8352b5e4f" +checksum = "0d72206cc3cb5acbb8cfe6653c1ea5322c26f7a53cad76f8970a31f20a823bdf" dependencies = [ "prost 0.13.5", "prost-types 0.13.5", @@ -2232,9 +2238,9 @@ dependencies = [ [[package]] name = "minimal_spongebob_community_neoeinstein-tonic" -version = "0.4.1-20251009015156-8bcd87750609.1" +version = "0.4.1-20251010205210-42ae8fd1517e.1" source = "sparse+https://buf.build/gen/cargo/" -checksum = "fa73a8386302f7d6aba6c519bbaacb9087e711d79d9496ec191b2c22bb2627b4" +checksum = "46f7ecbf0ff7510877a847108be91c9541ca392fddd5336875c3c09a9ea302fb" dependencies = [ "minimal_spongebob_community_neoeinstein-prost", "tonic 0.12.3", @@ -3162,7 +3168,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3709,7 +3715,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3718,7 +3724,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2111ef44dae28680ae9752bb89409e7310ca33a8c621ebe7b106cf5c928b3ac0" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4612,7 +4618,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/build-events-proto/Cargo.toml b/crates/build-events-proto/Cargo.toml index aecdd1c9c3..f05ca55363 100644 --- a/crates/build-events-proto/Cargo.toml +++ b/crates/build-events-proto/Cargo.toml @@ -19,8 +19,8 @@ tokio-stream = { version = "0.1", features = ["sync"] } spongebob = { path = "../spongebob", optional = true } # BSR proto dependencies for canonical spongebob.v1 types -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251009015156-8bcd87750609.1", registry = "buf" } -minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251009015156-8bcd87750609.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251010205210-42ae8fd1517e.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251010205210-42ae8fd1517e.1", registry = "buf" } [features] default = [] diff --git a/crates/build-events-proto/src/convert.rs b/crates/build-events-proto/src/convert.rs index 9df990c4fc..7d4e61265c 100644 --- a/crates/build-events-proto/src/convert.rs +++ b/crates/build-events-proto/src/convert.rs @@ -52,6 +52,9 @@ pub fn to_proto_build_event(event: &events::BuildEvent, invocation_id: &str) -> events::BuildEvent::ActionCompleted(e) => { proto::build_event::Event::ActionCompleted(to_proto_action_completed(e)) } + events::BuildEvent::BuildMetadata(e) => { + proto::build_event::Event::BuildMetadata(to_proto_build_metadata(e)) + } }; proto::BuildEvent { @@ -119,6 +122,12 @@ pub fn to_proto_action_completed(event: &events::ActionCompleted) -> proto::Acti } } +pub fn to_proto_build_metadata(event: &events::BuildMetadata) -> proto::BuildMetadata { + proto::BuildMetadata { + metadata: event.metadata.clone(), + } +} + pub fn to_proto_target_kind(kind: &events::TargetKind) -> proto::TargetKind { match kind { events::TargetKind::Binary => proto::TargetKind::Binary, @@ -155,6 +164,9 @@ pub fn from_proto_build_event( proto::build_event::Event::ActionCompleted(e) => Ok(events::BuildEvent::ActionCompleted( from_proto_action_completed(e)?, )), + proto::build_event::Event::BuildMetadata(e) => Ok(events::BuildEvent::BuildMetadata( + from_proto_build_metadata(e)?, + )), } } @@ -232,6 +244,14 @@ pub fn from_proto_action_completed( }) } +pub fn from_proto_build_metadata( + event: proto::BuildMetadata, +) -> Result { + Ok(events::BuildMetadata { + metadata: event.metadata, + }) +} + pub fn from_proto_target_kind( kind: proto::TargetKind, ) -> Result { diff --git a/crates/build-events-proto/src/service.rs b/crates/build-events-proto/src/service.rs index 05b508759a..8a063a1348 100644 --- a/crates/build-events-proto/src/service.rs +++ b/crates/build-events-proto/src/service.rs @@ -159,6 +159,7 @@ impl BuildEventService for BuildEventServiceImpl { Some(proto::build_event::Event::ActionCompleted(e)) => { e.timestamp_millis } + Some(proto::build_event::Event::BuildMetadata(_)) => 0, // Metadata events have no timestamp None => return None, }; diff --git a/crates/build-events-proto/src/subscriber.rs b/crates/build-events-proto/src/subscriber.rs index efc081071d..463361b81b 100644 --- a/crates/build-events-proto/src/subscriber.rs +++ b/crates/build-events-proto/src/subscriber.rs @@ -97,6 +97,7 @@ impl BuildEventSubscriber for GrpcStreamSubscriber { BuildEvent::TargetCompleted(_) => "", BuildEvent::ActionStarted(_) => "", BuildEvent::ActionCompleted(_) => "", + BuildEvent::BuildMetadata(_) => "", // Metadata doesn't have invocation_id in inner fields }; // Convert Rust event to proto using ergonomic extension trait diff --git a/crates/build-events/src/events.rs b/crates/build-events/src/events.rs index df7ec2be17..b61d79741c 100644 --- a/crates/build-events/src/events.rs +++ b/crates/build-events/src/events.rs @@ -5,6 +5,7 @@ //! for easy serialization to JSON, protobuf, or other formats. use serde::{Deserialize, Serialize}; +use std::collections::HashMap; /// Main build event enum that wraps all possible event types #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] @@ -22,6 +23,8 @@ pub enum BuildEvent { ActionStarted(ActionStarted), /// Action completed executing ActionCompleted(ActionCompleted), + /// Build metadata (user, branch, commit, etc.) + BuildMetadata(BuildMetadata), } /// Event emitted when a build starts @@ -112,6 +115,14 @@ pub struct ActionCompleted { pub stderr: Option, } +/// Event containing arbitrary key-value metadata about the build environment +/// Standard keys: "user", "branch", "commit", "repo_url" +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct BuildMetadata { + /// Metadata key-value pairs + pub metadata: HashMap, +} + /// Type of build target #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "snake_case")] diff --git a/crates/build-events/src/subscribers/logger.rs b/crates/build-events/src/subscribers/logger.rs index d8af556794..3313494ba8 100644 --- a/crates/build-events/src/subscribers/logger.rs +++ b/crates/build-events/src/subscribers/logger.rs @@ -107,6 +107,13 @@ impl BuildEventSubscriber for LoggerSubscriber { "Action completed" ); } + BuildEvent::BuildMetadata(e) => { + info!( + event = "build_metadata", + metadata = ?e.metadata, + "Build metadata" + ); + } } Ok(()) diff --git a/crates/build-sandbox/Cargo.toml b/crates/build-sandbox/Cargo.toml index 36f332befb..48171a5d81 100644 --- a/crates/build-sandbox/Cargo.toml +++ b/crates/build-sandbox/Cargo.toml @@ -19,4 +19,4 @@ common.workspace = true hakoniwa.workspace = true # TODO, get this out of the build-sandbox crate -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251009015156-8bcd87750609.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251010205210-42ae8fd1517e.1", registry = "buf" } diff --git a/crates/minimal/Cargo.toml b/crates/minimal/Cargo.toml index ce8e71b375..b17dce4dda 100644 --- a/crates/minimal/Cargo.toml +++ b/crates/minimal/Cargo.toml @@ -57,4 +57,5 @@ size.workspace = true object.workspace = true globset.workspace = true +[target.'cfg(target_os = "linux")'.dependencies] hakoniwa.workspace = true diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index 4469bfa2a4..9b44b5bbae 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -1,12 +1,14 @@ use crate::{Error, lockfile::PrebuiltsLock, remote_storage::RemoteStorage, run::Run}; use crate::{GlobalArgs, PackagesArg}; use anyhow::Context; -use build_events::events::{BuildEvent, BuildFinished, BuildStarted, current_millis}; +use build_events::events::{BuildEvent, BuildFinished, BuildMetadata, BuildStarted, current_millis}; use build_events::{BuildEventBus, BuildEventDispatcher}; use build_events_proto::SpongeBobSubscriberV2; use cache::{Cache, CacheBinProvider, LocalDir, RemoteBinProvider}; use graph::{DepGraph, ExecPlan, Transitives}; +use std::collections::HashMap; use std::path::Path; +use std::process::Command; use tracing::info; #[derive(Debug, clap::Args)] @@ -97,6 +99,50 @@ pub async fn cmd_build_impl( working_directory, })); + // Collect and emit git metadata + let mut metadata = HashMap::new(); + + // Get git user + if let Ok(output) = Command::new("git").args(["config", "user.name"]).output() { + if output.status.success() { + if let Ok(user) = String::from_utf8(output.stdout) { + metadata.insert("user".to_string(), user.trim().to_string()); + } + } + } + + // Get git branch + if let Ok(output) = Command::new("git").args(["rev-parse", "--abbrev-ref", "HEAD"]).output() { + if output.status.success() { + if let Ok(branch) = String::from_utf8(output.stdout) { + metadata.insert("branch".to_string(), branch.trim().to_string()); + } + } + } + + // Get git commit SHA + if let Ok(output) = Command::new("git").args(["rev-parse", "HEAD"]).output() { + if output.status.success() { + if let Ok(commit) = String::from_utf8(output.stdout) { + metadata.insert("commit".to_string(), commit.trim().to_string()); + } + } + } + + // Get git remote URL + if let Ok(output) = Command::new("git").args(["config", "remote.origin.url"]).output() { + if output.status.success() { + if let Ok(repo_url) = String::from_utf8(output.stdout) { + metadata.insert("repo_url".to_string(), repo_url.trim().to_string()); + } + } + } + + // Emit BuildMetadata event if we collected any metadata + if !metadata.is_empty() { + event_bus.emit(BuildEvent::BuildMetadata(BuildMetadata { metadata })); + } + let build_success = match (globals.no_cache, globals.no_fetch) { // No local or remote cache (true, true) => { diff --git a/crates/minimal/src/main.rs b/crates/minimal/src/main.rs index fcae645da0..e64cc0ea7d 100644 --- a/crates/minimal/src/main.rs +++ b/crates/minimal/src/main.rs @@ -36,7 +36,9 @@ mod cmd_upload_cache; use cmd_upload_cache::{UploadArgs, cmd_upload_cache}; mod cmd_patched_build; use cmd_patched_build::{PatchedBuildArgs, cmd_patched_build}; +#[cfg(target_os = "linux")] mod cmd_run; +#[cfg(target_os = "linux")] use cmd_run::{RunArgs, cmd_run}; #[derive(Parser)] @@ -68,6 +70,7 @@ enum Command { #[clap(hide = !std::env::var("MINIMAL_SCIENCE_MODE").is_ok())] PatchedBuild(PatchedBuildArgs), /// Runs a command using the given packages, in the current working directory. + #[cfg(target_os = "linux")] #[clap(hide = !std::env::var("MINIMAL_SCIENCE_MODE").is_ok())] Run(RunArgs), } @@ -354,6 +357,7 @@ async fn main() -> Result<()> { Command::NewWorldUpdate(args) => cmd_new_world_update(args, &cli.global_args).await, Command::OciImage(args) => cmd_oci_image(args, &cli.global_args).await, Command::PatchedBuild(args) => cmd_patched_build(args, &cli.global_args).await, + #[cfg(target_os = "linux")] Command::Run(args) => cmd_run(args, &cli.global_args).await, }; diff --git a/crates/spongebob/Cargo.toml b/crates/spongebob/Cargo.toml index 346a22972b..6859528c59 100644 --- a/crates/spongebob/Cargo.toml +++ b/crates/spongebob/Cargo.toml @@ -5,8 +5,8 @@ edition.workspace = true publish.workspace = true [dependencies] -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251009015156-8bcd87750609.1", registry = "buf" } -minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251009015156-8bcd87750609.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251010205210-42ae8fd1517e.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251010205210-42ae8fd1517e.1", registry = "buf" } thiserror.workspace = true tonic = { version = "0.12.3", features = ["tls", "tls-native-roots"] } tracing.workspace = true From 37e251aa9d4f5084e336531b1b5303fabd7adc3b Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Tue, 14 Oct 2025 09:49:38 -0700 Subject: [PATCH 10/39] rm log_build_results_to_spongebob --- crates/minimal/src/cmd_build.rs | 54 --------------------------------- 1 file changed, 54 deletions(-) diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index 9b44b5bbae..64a9cda1f2 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -249,11 +249,6 @@ pub async fn cmd_build_impl( } } - // Log build results to SpongeBob - if let Some(ref mut invocation) = spongebob_invocation { - log_build_results_to_spongebob(invocation, graph, build_succeeded).await?; - } - let spongebob_url = spongebob_invocation .as_ref() .map(|inv| inv.url().to_string()); @@ -264,55 +259,6 @@ pub async fn cmd_build_impl( Ok(()) } -/// Log build command results to SpongeBob -async fn log_build_results_to_spongebob( - spongebob_invocation: &mut spongebob::SpongeBobInvocation, - graph: &DepGraph, - success: bool, -) -> anyhow::Result<()> { - let packages_list = graph - .top_levels - .iter() - .map(|bsr| graph.get(bsr).unwrap().name.as_str()) - .collect::>() - .join(", "); - - let _build_summary = if success { - format!( - "Build completed successfully for packages: {}", - packages_list - ) - } else { - format!("Build failed for packages: {}", packages_list) - }; - - let build_log = format!( - "Build Command Summary\n\ - Packages: {}\n\ - Status: {}\n\ - Total packages: {}\n", - packages_list, - if success { "SUCCESS" } else { "FAILED" }, - graph.top_levels.len() - ); - - // Use a semantic target ID for the build command summary - let build_command_target_id = "build-summary".to_string(); - - // Upload build summary to the synthetic target - info!("Uploading build summary to SpongeBob invocation"); - spongebob_invocation - .upload_file( - &build_command_target_id, - "build-summary.txt", - build_log.into_bytes(), - ) - .await - .map_err(|e| anyhow::anyhow!("Failed to upload build summary to SpongeBob: {}", e))?; - - Ok(()) -} - /// Display a summary of what was built and where outputs can be found fn display_build_summary( graph: &DepGraph, From 3c7e35af57349cf4dab46ae9159657a63ed02714 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Tue, 14 Oct 2025 11:15:18 -0700 Subject: [PATCH 11/39] can I commit the token?? --- .cargo/config.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/.cargo/config.toml b/.cargo/config.toml index aa60b61395..6287b023da 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,6 +1,7 @@ [registries.buf] index = "sparse+https://buf.build/gen/cargo/" credential-provider = "cargo:token" +token = "Bearer a54105a33eb0bc75fd05666e96f6a63ec8bb54fa7b964665e230d24824268687" [build] rustflags = ["-C", "prefer-dynamic"] From 0969c6ca17571bbc8110aedf6b446dbef346482e Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Tue, 14 Oct 2025 11:58:48 -0700 Subject: [PATCH 12/39] fix tests --- crates/build-events-proto/src/lib.rs | 14 +++++--------- crates/build-events-proto/src/service.rs | 5 ++++- .../src/spongebob_subscriber_v2.rs | 4 ++-- crates/build-events/src/lib.rs | 1 + 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/build-events-proto/src/lib.rs b/crates/build-events-proto/src/lib.rs index bc13418386..ff802fab31 100644 --- a/crates/build-events-proto/src/lib.rs +++ b/crates/build-events-proto/src/lib.rs @@ -65,7 +65,7 @@ //! ```no_run //! use build_events_proto::proto::{ //! build_event_service_client::BuildEventServiceClient, -//! StreamEventsRequest, +//! StreamBuildEventsRequest, //! }; //! use tokio_stream::StreamExt; //! @@ -73,12 +73,12 @@ //! async fn main() -> Result<(), Box> { //! let mut client = BuildEventServiceClient::connect("http://[::1]:50051").await?; //! -//! let request = StreamEventsRequest { +//! let request = StreamBuildEventsRequest { //! invocation_id: None, //! start_timestamp_millis: None, //! }; //! -//! let mut stream = client.stream_events(request).await?.into_inner(); +//! let mut stream = client.stream_build_events(request).await?.into_inner(); //! //! while let Some(event) = stream.next().await { //! let event = event?; @@ -96,7 +96,7 @@ //! ``` //! use build_events::events::{BuildStarted, current_millis}; //! use build_events::BuildEvent; -//! use build_events_proto::proto; +//! use build_events_proto::{proto, convert::ToProto}; //! //! // Rust -> Proto //! let rust_event = BuildEvent::BuildStarted(BuildStarted { @@ -106,11 +106,7 @@ //! working_directory: "/tmp".to_string(), //! }); //! -//! let proto_event: proto::BuildEvent = rust_event.clone().into(); -//! -//! // Proto -> Rust -//! let converted_back: BuildEvent = proto_event.try_into().unwrap(); -//! assert_eq!(rust_event, converted_back); +//! let proto_event: proto::BuildEvent = rust_event.to_proto("test-123"); //! ``` // Use BSR-generated protobuf code from canonical spongebob.v1 package diff --git a/crates/build-events-proto/src/service.rs b/crates/build-events-proto/src/service.rs index 8a063a1348..9527ebbca8 100644 --- a/crates/build-events-proto/src/service.rs +++ b/crates/build-events-proto/src/service.rs @@ -70,7 +70,10 @@ impl BuildEventServiceImpl { /// use build_events_proto::{BuildEventServiceImpl, proto}; /// /// let service = BuildEventServiceImpl::new(1000); - /// let event = proto::BuildEvent { event: None }; + /// let event = proto::BuildEvent { + /// invocation_id: String::new(), + /// event: None + /// }; /// service.publish(event); /// ``` pub fn publish(&self, event: proto::BuildEvent) { diff --git a/crates/build-events-proto/src/spongebob_subscriber_v2.rs b/crates/build-events-proto/src/spongebob_subscriber_v2.rs index 3056017606..6765353d08 100644 --- a/crates/build-events-proto/src/spongebob_subscriber_v2.rs +++ b/crates/build-events-proto/src/spongebob_subscriber_v2.rs @@ -24,7 +24,7 @@ use crate::convert::ToProto; /// async fn main() -> Result<(), Box> { /// // Create a spongebob client and invocation /// let mut spongebob = spongebob::SpongeBob::new().await?; -/// let invocation = spongebob.create_invocation("my-build").await?; +/// let invocation = spongebob.create_invocation(); /// /// // Create subscriber from the invocation /// let subscriber = SpongeBobSubscriberV2::from_invocation(invocation); @@ -52,7 +52,7 @@ impl SpongeBobSubscriberV2 { /// # use build_events_proto::SpongeBobSubscriberV2; /// # async fn example() -> Result<(), Box> { /// let mut spongebob = spongebob::SpongeBob::new().await?; - /// let invocation = spongebob.create_invocation("my-build").await?; + /// let invocation = spongebob.create_invocation(); /// let subscriber = SpongeBobSubscriberV2::from_invocation(invocation); /// # Ok(()) /// # } diff --git a/crates/build-events/src/lib.rs b/crates/build-events/src/lib.rs index b5ff600919..cdcb1606b3 100644 --- a/crates/build-events/src/lib.rs +++ b/crates/build-events/src/lib.rs @@ -54,6 +54,7 @@ //! // Emit events from build //! event_bus.emit(BuildEvent::BuildStarted(BuildStarted { //! invocation_id: "build-123".to_string(), +//! command: "cargo build".to_string(), //! timestamp_millis: current_millis(), //! working_directory: std::env::current_dir() //! .unwrap() From e7f490a506cdf9f8d50dbf4d586e33112842ed57 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Tue, 14 Oct 2025 12:12:37 -0700 Subject: [PATCH 13/39] remove example --- .cargo/config.toml | 3 - BUILD_EVENTS_INTEGRATION.md | 331 ------------------- BUILD_EVENT_DESIGN.md | 339 -------------------- crates/build-events/examples/basic_usage.rs | 165 ---------- deny.toml | 20 +- flake.nix | 3 +- 6 files changed, 11 insertions(+), 850 deletions(-) delete mode 100644 BUILD_EVENTS_INTEGRATION.md delete mode 100644 BUILD_EVENT_DESIGN.md delete mode 100644 crates/build-events/examples/basic_usage.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 6287b023da..c9ba2da727 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,6 +2,3 @@ index = "sparse+https://buf.build/gen/cargo/" credential-provider = "cargo:token" token = "Bearer a54105a33eb0bc75fd05666e96f6a63ec8bb54fa7b964665e230d24824268687" - -[build] -rustflags = ["-C", "prefer-dynamic"] diff --git a/BUILD_EVENTS_INTEGRATION.md b/BUILD_EVENTS_INTEGRATION.md deleted file mode 100644 index e761e0cf48..0000000000 --- a/BUILD_EVENTS_INTEGRATION.md +++ /dev/null @@ -1,331 +0,0 @@ -# Build Events Integration - ✅ COMPLETE - -## Overview -Successfully integrated the build events system from minpkgs into spongebob for persistent storage and remote observability. The integration is now fully wired into the minpkgs CLI and emits build lifecycle events. - -## ✅ What Was Implemented - -### 1. Spongebob Backend (Go) - -#### Database Schema (schema.sql:28-38) -```sql -CREATE TABLE build_events ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - invocation_id UUID NOT NULL, - event_type VARCHAR(50) NOT NULL, - timestamp_millis BIGINT NOT NULL, - event_data JSONB NOT NULL, - created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), - FOREIGN KEY (invocation_id) REFERENCES invocations(id) ON DELETE CASCADE -); -``` - -With indexes on `invocation_id`, `timestamp_millis`, and `event_type`. - -#### SQL Queries (queries.sql:35-54) -- `CreateBuildEvent` - Insert event with JSONB storage -- `ListBuildEventsByInvocation` - Query by invocation with pagination -- `ListBuildEventsByTimeRange` - Query by timestamp range -- `GetBuildEvent` - Retrieve single event by ID - -#### Protobuf Definitions -**proto/spongebob/v1/build_events.proto** (new file): -- `BuildEvent` with oneof for all event types -- `BuildStarted`, `BuildFinished`, `TargetStarted`, `TargetCompleted`, `ActionStarted`, `ActionCompleted` -- `TargetKind` enum - -**proto/spongebob/v1/service.proto** (extended): -- `PublishBuildEvent(PublishBuildEventRequest) → PublishBuildEventResponse` -- `ListBuildEvents(ListBuildEventsRequest) → ListBuildEventsResponse` -- `StreamBuildEvents(StreamBuildEventsRequest) → stream BuildEvent` - -Published to BSR: `buf.build/minimal/spongebob:302595534f7d` - -#### Go Service Implementation (internal/server/grpc.go) - -**PublishBuildEvent RPC** (grpc.go:319-374): -- Validates invocation exists -- Extracts event type and timestamp -- Marshals to JSON for storage -- Broadcasts to streaming clients -- Returns resource name: `invocations/{id}/events/{event-id}` - -**ListBuildEvents RPC** (grpc.go:376-448): -- Paginated event listing (default 50, max 100) -- Base64-encoded page tokens -- Unmarshals JSONB to proto -- Returns events with resource names - -**StreamBuildEvents RPC** (grpc.go:450-507): -- Real-time server-side streaming -- Optional invocation_id and timestamp filtering -- In-memory broadcast to all connected clients -- Non-blocking sends (skips slow clients) - -**Helper Functions** (grpc.go:583-651): -- `getEventTypeAndTimestamp()` - Extract from proto oneofs -- `getEventInvocationID()` - Extract invocation from events -- `broadcastBuildEvent()` - Broadcast to stream subscribers -- `generateClientID()` - Unique stream client IDs -- `encodePageToken()` / `decodePageToken()` - Pagination - -### 2. Minpkgs Client (Rust) - -#### Build Events Proto Crate - -**spongebob_convert.rs** (feature-gated): -- `to_proto_build_event()` - Main conversion function -- Separate converters for each event type -- Converts Rust enums to proto i32 values -- Zero-copy where possible, clones only when needed - -**spongebob_subscriber_v2.rs** (feature-gated): -```rust -pub struct SpongeBobSubscriberV2 { - invocation: Arc>, -} - -impl BuildEventSubscriber for SpongeBobSubscriberV2 { - async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { - let proto_event = to_proto_build_event(event); - let mut invocation = self.invocation.lock().await; - invocation.publish_build_event(proto_event).await?; - Ok(()) - } -} -``` - -**Feature Flags** (Cargo.toml): -- `spongebob-subscriber` feature gates spongebob integration -- Optional dependency on `spongebob` crate -- Optional BSR proto import for conversion - -#### Spongebob Crate Updates - -**lib.rs:50-85** - New method on `SpongeBobInvocation`: -```rust -pub async fn publish_build_event(&mut self, event: BuildEvent) -> Result<()> { - let request = PublishBuildEventRequest { - parent: self.resource_name.clone(), - event: Some(event), - }; - self.service.publish_build_event(request).await?; - Ok(()) -} -``` - -## 🏗️ Architecture - -``` -┌─────────────────────────────────────────────────────┐ -│ Minpkgs Build │ -│ │ -│ BuildEventBus → BuildEventDispatcher │ -│ ↓ │ -│ SpongeBobSubscriberV2 │ -└─────────────────────┬───────────────────────────────┘ - │ gRPC (PublishBuildEvent) - ↓ -┌─────────────────────────────────────────────────────┐ -│ Spongebob Service (Go) │ -│ │ -│ PublishBuildEvent RPC │ -│ ↓ │ -│ PostgreSQL (JSONB storage) │ -│ ↓ │ -│ Broadcast to streaming clients │ -│ │ -│ ListBuildEvents RPC ←── dash/API clients │ -│ StreamBuildEvents RPC ←── real-time UI │ -└─────────────────────────────────────────────────────┘ -``` - -## 📝 Usage Example - -### In minpkgs build: - -```rust -use build_events::{BuildEventBus, BuildEventDispatcher}; -use build_events_proto::SpongeBobSubscriberV2; - -#[tokio::main] -async fn main() -> Result<()> { - // Connect to spongebob - let mut spongebob = spongebob::SpongeBob::new().await?; - let invocation = spongebob.create_invocation("my-build").await?; - - println!("Build logs: {}", invocation.url()); - - // Setup build events - let event_bus = BuildEventBus::new(10000); - - let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); - dispatcher.add_subscriber(Box::new( - SpongeBobSubscriberV2::from_invocation(invocation) - )); - - tokio::spawn(async move { - dispatcher.run().await; - }); - - // Events are automatically published to spongebob - event_bus.emit(BuildEvent::BuildStarted(BuildStarted { - invocation_id: "test-123".to_string(), - command_line: vec!["cargo".to_string(), "build".to_string()], - timestamp_millis: current_millis(), - working_directory: env::current_dir()?.to_string_lossy().to_string(), - })); - - // ... run build ... - - Ok(()) -} -``` - -## ✅ Testing - -All components build successfully: -- ✅ Spongebob Go service: `go build -o spongebob .` -- ✅ Minpkgs workspace: `cargo build` -- ✅ Build-events-proto with spongebob-subscriber: `cargo build --features spongebob-subscriber` -- ✅ Minpkgs CLI with build events: `cargo build --package minimal` - -To test the end-to-end integration: -```bash -cd minpkgs -cargo run -- build --package bash -# Events will be automatically published to spongebob.minimal.farm -# Check the invocation URL printed at the end -``` - -### 3. Minpkgs CLI Integration - -#### Updated Files -**crates/minimal/Cargo.toml** (lines 10-11): -- Added `build-events.workspace = true` -- Added `build-events-proto = { path = "../build-events-proto", features = ["spongebob-subscriber"] }` - -**crates/minimal/src/cmd_build.rs**: - -**Imports** (lines 4-6): -```rust -use build_events::events::{BuildEvent, BuildFinished, BuildStarted, current_millis}; -use build_events::{BuildEventBus, BuildEventDispatcher}; -use build_events_proto::SpongeBobSubscriberV2; -``` - -**Build Events Setup** (lines 74-110): -```rust -// Setup build events system -let event_bus = BuildEventBus::new(10000); - -// Create dispatcher with SpongeBobSubscriberV2 if we have an invocation -if let Some(ref invocation) = spongebob_invocation { - let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); - let subscriber = SpongeBobSubscriberV2::from_invocation(invocation.clone()); - dispatcher.add_subscriber(Box::new(subscriber)); - - // Spawn dispatcher in background - tokio::spawn(async move { - dispatcher.run().await; - }); -} - -// Get invocation_id for events -let invocation_id = spongebob_invocation - .as_ref() - .map(|inv| inv.resource_name().to_string()) - .unwrap_or_else(|| "local".to_string()); - -// Get command line for BuildStarted event -let command_line = std::env::args().collect::>(); -let working_directory = std::env::current_dir() - .unwrap_or_default() - .to_string_lossy() - .to_string(); - -// Emit BuildStarted event -event_bus.emit(BuildEvent::BuildStarted(BuildStarted { - invocation_id: invocation_id.clone(), - command_line, - timestamp_millis: current_millis(), - working_directory, -})); -``` - -**Build Execution with Event Emission** (lines 153-166): -```rust -// Determine if build was successful -let build_succeeded = build_success.is_ok(); -let error_message = build_success.as_ref().err().map(|e| e.to_string()); - -// Emit BuildFinished event -event_bus.emit(BuildEvent::BuildFinished(BuildFinished { - invocation_id: invocation_id.clone(), - success: build_succeeded, - timestamp_millis: current_millis(), - error_message, -})); - -// Propagate error if build failed -build_success.context("Failed to execute build")?; -``` - -## 🚀 Deployment Steps - -To deploy this integration: - -1. **Deploy Database Migration** - Run schema.sql on production database - -2. **Deploy Spongebob Service** - Deploy updated Go service with new RPCs - -3. **Deploy Minpkgs Binary** - Build and deploy minpkgs with the new integration: - ```bash - cd minpkgs - cargo build --release - ``` - -4. **Test End-to-End**: - - Run minpkgs build with spongebob integration - - Verify events in PostgreSQL: `SELECT * FROM build_events;` - - Test ListBuildEvents API - - Test StreamBuildEvents with grpcurl or dash UI - -## 📦 Files Modified - -### Spongebob (Go) -- `schema.sql` - Added build_events table -- `queries.sql` - Added 4 queries -- `proto/spongebob/v1/build_events.proto` - New file -- `proto/spongebob/v1/service.proto` - Added 3 RPCs -- `internal/server/grpc.go` - ~350 lines of implementation -- `internal/db/models.go` - Generated BuildEvent model -- `internal/db/queries.sql.go` - Generated query functions - -### Minpkgs (Rust) -- `crates/build-events-proto/Cargo.toml` - Added spongebob-subscriber feature -- `crates/build-events-proto/src/lib.rs` - Feature-gated exports -- `crates/build-events-proto/src/spongebob_convert.rs` - New file -- `crates/build-events-proto/src/spongebob_subscriber_v2.rs` - New file -- `crates/spongebob/Cargo.toml` - Updated to latest proto -- `crates/spongebob/src/lib.rs` - Added publish_build_event method -- `crates/minimal/Cargo.toml` - Added build-events and build-events-proto dependencies -- `crates/minimal/src/cmd_build.rs` - Wired BuildEventBus, dispatcher, and event emission - -## 🎯 Key Design Decisions - -1. **JSONB Storage** - Flexible schema evolution without migrations -2. **Event Type Column** - Enables fast filtering without parsing JSON -3. **Timestamp Index** - Supports time-range queries for analytics -4. **Non-blocking Broadcasts** - Slow stream clients don't affect ingestion -5. **Feature Flag** - Optional spongebob integration doesn't force dependency -6. **Arc>** - Thread-safe sharing for async subscriber - -## 🔒 Production Considerations - -- **Rate Limiting** - Consider limiting PublishBuildEvent requests per invocation -- **Event Size Limits** - Current max JSONB size should be documented -- **Stream Client Limits** - Monitor number of concurrent StreamBuildEvents connections -- **Database Performance** - Monitor JSONB query performance at scale -- **Error Handling** - Subscriber logs warnings but doesn't fail builds on errors diff --git a/BUILD_EVENT_DESIGN.md b/BUILD_EVENT_DESIGN.md deleted file mode 100644 index 8bec45dc6c..0000000000 --- a/BUILD_EVENT_DESIGN.md +++ /dev/null @@ -1,339 +0,0 @@ -# Build Events System Design Proposal (Revised) - -## Overview -Create a dedicated build event protocol (BEP) system for `minimal` that enables remote observability of build execution. This system will be separate from diagnostic tracing and will emit structured, strongly-typed events that can be consumed by local and remote services. - -## Goals -- Separate build events from diagnostic logs (`tracing`) -- Provide structured, serializable Rust events -- Enable multiple concurrent subscribers (local UI, remote services, file writers) -- Maintain low overhead and back-pressure handling -- Allow subscribers to choose their own serialization format - -## Crate Structure - -**Crate name**: `build-events` - -**Location**: `crates/build-events/` - -**Dependencies**: -```toml -[dependencies] -tokio = { version = "1", features = ["sync", "time"] } -serde = { version = "1", features = ["derive"] } -thiserror = "1" -async-trait = "0.1" - -[dev-dependencies] -tokio = { version = "1", features = ["full", "test-util"] } -serde_json = "1" -``` - -## Architecture - -### Core Components - -1. **Event Types** (Pure Rust) - - Strongly-typed event enums representing build lifecycle - - Derive `Clone`, `Debug`, `Serialize`, `Deserialize` - - Minimal initial set: BuildStarted, TargetStarted, TargetCompleted, ActionStarted, ActionCompleted, BuildFinished - -2. **BuildEventBus** (publisher) - - Central event dispatcher using `tokio::sync::broadcast` - - Thread-safe, can be cloned and passed to any component - - Non-blocking emission with configurable buffer size - -3. **BuildEventSubscriber** (trait) - - Similar to `tracing::Subscriber` - - Allows pluggable event consumers - - Async trait for handling events - - **Subscribers handle their own serialization** (JSON, protobuf, bincode, etc.) - -4. **Event Stream Handler** - - Manages subscriber registration - - Distributes events from broadcast channel to subscribers - - Handles subscriber lifecycle - -## API Design - -### Event Emission (Producer Side) - -```rust -// Initialize at build start -let event_bus = BuildEventBus::new(capacity: 10000); - -// Clone and use anywhere in the codebase -let bus = event_bus.clone(); -bus.emit(BuildEvent::BuildStarted { - invocation_id: uuid::Uuid::new_v4().to_string(), - command_line: args, - timestamp_millis: current_millis(), - working_directory: env::current_dir().unwrap(), -}); - -// Emit from any component -bus.emit(BuildEvent::TargetStarted { - label: "//foo:bar".to_string(), - target_kind: TargetKind::Binary, - timestamp_millis: current_millis(), -}); -``` - -### Event Consumption (Consumer Side) - -```rust -#[async_trait] -pub trait BuildEventSubscriber: Send + Sync { - async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError>; - - fn name(&self) -> &str; - - // Optional: called when subscriber is dropped - async fn on_close(&self) -> Result<(), SubscriberError> { - Ok(()) - } -} - -// Register subscribers -let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); -dispatcher.add_subscriber(Box::new(JsonFileWriter::new("build_events.jsonl"))); -dispatcher.add_subscriber(Box::new(TerminalUI::new())); - -// Start processing (runs until bus is closed) -tokio::spawn(async move { - dispatcher.run().await; -}); -``` - -## Event Schema (Pure Rust) - -```rust -use serde::{Serialize, Deserialize}; - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum BuildEvent { - BuildStarted(BuildStarted), - BuildFinished(BuildFinished), - TargetStarted(TargetStarted), - TargetCompleted(TargetCompleted), - ActionStarted(ActionStarted), - ActionCompleted(ActionCompleted), -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct BuildStarted { - pub invocation_id: String, - pub command_line: Vec, - pub timestamp_millis: i64, - pub working_directory: String, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct BuildFinished { - pub invocation_id: String, - pub success: bool, - pub timestamp_millis: i64, - pub error_message: Option, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct TargetStarted { - pub label: String, - pub target_kind: TargetKind, - pub timestamp_millis: i64, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct TargetCompleted { - pub label: String, - pub success: bool, - pub timestamp_millis: i64, - pub error_message: Option, - pub outputs: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ActionStarted { - pub action_id: String, - pub label: String, - pub mnemonic: String, - pub timestamp_millis: i64, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ActionCompleted { - pub action_id: String, - pub success: bool, - pub timestamp_millis: i64, - pub exit_code: i32, - pub stdout: Option, - pub stderr: Option, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TargetKind { - Binary, - Library, - Test, -} -``` - -## Module Structure - -``` -crates/build-events/ -├── Cargo.toml -└── src/ - ├── lib.rs # Public API exports - ├── bus.rs # BuildEventBus implementation - ├── subscriber.rs # BuildEventSubscriber trait - ├── dispatcher.rs # Event distribution logic - ├── events.rs # Event type definitions - └── subscribers/ # Built-in subscriber implementations - ├── mod.rs - ├── logger.rs # Logs events via tracing (for debugging) - └── json_file.rs # Writes JSON-lines to file -``` - -## Future: Protobuf Subscriber Layer - -When you need gRPC support, create a separate crate: - -``` -crates/build-events-proto/ -├── Cargo.toml -├── build.rs # prost-build configuration -├── proto/ -│ └── events.proto # Protobuf definitions -└── src/ - ├── lib.rs - ├── convert.rs # Conversions: BuildEvent <-> proto - └── subscriber.rs # GrpcStreamSubscriber implementation -``` - -The proto crate would: -- Define protobuf schema -- Implement `From` for proto types -- Provide a `GrpcStreamSubscriber` that converts and streams events - -## Integration with Minimal - -### 1. Workspace Configuration -Add to root `Cargo.toml`: -```toml -[workspace] -members = [ - "crates/build-events", - # ... other crates -] -``` - -### 2. Usage in Main Binary - -```rust -use build_events::{BuildEventBus, BuildEventDispatcher, BuildEvent, subscribers}; - -#[tokio::main] -async fn main() { - // Initialize tracing (existing diagnostic logs) - tracing_subscriber::fmt::init(); - - // Initialize build events - let event_bus = BuildEventBus::new(10000); - - // Setup subscribers - let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); - dispatcher.add_subscriber(Box::new( - subscribers::JsonFileWriter::new("build_events.jsonl").unwrap() - )); - - // Start event processing - let dispatcher_handle = tokio::spawn(async move { - dispatcher.run().await - }); - - // Pass event_bus to build executor - let build_result = run_build(event_bus.clone(), build_config).await; - - // Shutdown: drop event_bus, wait for dispatcher to drain - drop(event_bus); - let _ = dispatcher_handle.await; - - // Continue with result handling... -} -``` - -### 3. Emitting Events in Build Logic - -```rust -async fn execute_target(bus: BuildEventBus, target: &Target) -> Result<()> { - bus.emit(BuildEvent::TargetStarted(TargetStarted { - label: target.label.clone(), - target_kind: target.kind, - timestamp_millis: current_millis(), - })); - - // ... build logic ... - - bus.emit(BuildEvent::TargetCompleted(TargetCompleted { - label: target.label.clone(), - success: result.is_ok(), - timestamp_millis: current_millis(), - error_message: result.as_ref().err().map(|e| e.to_string()), - outputs: result.as_ref().ok().cloned().unwrap_or_default(), - })); - - result -} -``` - -## Implementation Phases - -### Phase 1: Core Infrastructure -- [ ] Setup `build-events` crate with serde -- [ ] Define Rust event types (6 event variants) -- [ ] Implement `BuildEventBus` with broadcast channel -- [ ] Define `BuildEventSubscriber` trait -- [ ] Implement `BuildEventDispatcher` -- [ ] Write unit tests for bus and dispatcher - -### Phase 2: Built-in Subscribers -- [ ] Implement `LoggerSubscriber` (emits to tracing for debugging) -- [ ] Implement `JsonFileWriter` (JSON-lines format) -- [ ] Add integration tests - -### Phase 3: Integration -- [ ] Add to `minimal` workspace -- [ ] Wire into main binary -- [ ] Add event emission to key build phases -- [ ] Documentation and examples - -### Phase 4: Future (Out of Scope) -- `build-events-proto` crate with protobuf definitions -- gRPC streaming subscriber -- Event filtering/sampling -- Additional event types -- Metrics/analytics subscriber - -## Error Handling - -- **Bus emission**: Best-effort, never blocks. If channel is full, events are dropped (lagging subscribers) -- **Subscriber errors**: Logged via `tracing::warn!`, but don't stop other subscribers -- **Backpressure**: Broadcast channel size is configurable; slow subscribers will lag and miss events - -## Testing Strategy - -- Unit tests for event bus mechanics -- Unit tests for dispatcher subscription management -- Integration tests with mock subscribers -- Benchmark tests for overhead measurement -- Example subscriber implementations as test fixtures - -## Open Questions / Future Considerations - -1. Should we add event sequence numbers for ordering guarantees? -2. Do we need event filtering at the bus level? -3. Should the JSON file writer support rotation/compression? -4. Metrics on dropped events? diff --git a/crates/build-events/examples/basic_usage.rs b/crates/build-events/examples/basic_usage.rs deleted file mode 100644 index 49f6567990..0000000000 --- a/crates/build-events/examples/basic_usage.rs +++ /dev/null @@ -1,165 +0,0 @@ -//! Basic usage example for the build-events crate -//! -//! This example demonstrates: -//! - Creating a BuildEventBus -//! - Setting up multiple subscribers (LoggerSubscriber and JsonFileWriter) -//! - Emitting all event types -//! - Graceful shutdown - -use build_events::{ - BuildEvent, BuildEventBus, BuildEventDispatcher, - events::{ - ActionCompleted, ActionStarted, BuildFinished, BuildStarted, TargetCompleted, TargetKind, - TargetStarted, current_millis, - }, - subscribers::{JsonFileWriter, LoggerSubscriber}, -}; - -#[tokio::main] -async fn main() { - // Initialize tracing for LoggerSubscriber - tracing_subscriber::fmt() - .with_target(false) - .with_thread_ids(false) - .with_file(false) - .with_line_number(false) - .init(); - - println!("=== Build Events System Demo ===\n"); - - // Create event bus with capacity for 10,000 events - let event_bus = BuildEventBus::new(10000); - println!("Created BuildEventBus with capacity 10,000"); - - // Setup subscribers - let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); - - // Add LoggerSubscriber to emit events to tracing - dispatcher.add_subscriber(Box::new(LoggerSubscriber::new())); - - // Add JsonFileWriter to write events to a file - let json_path = "build_events.jsonl"; - dispatcher.add_subscriber(Box::new( - JsonFileWriter::new(json_path).expect("Failed to create JSON file writer"), - )); - - println!("Registered subscribers: LoggerSubscriber, JsonFileWriter"); - println!("Events will be written to: {}\n", json_path); - - // Start event processing in background - let dispatcher_handle = tokio::spawn(async move { - dispatcher.run().await; - }); - - println!("Started event dispatcher\n"); - - // Simulate a build by emitting events - simulate_build(&event_bus).await; - - println!("\nBuild simulation complete"); - - // Shutdown: drop event_bus to close channel, wait for dispatcher to drain - println!("Shutting down event bus..."); - drop(event_bus); - - // Wait for dispatcher to finish processing - dispatcher_handle.await.expect("Dispatcher task failed"); - - println!("Event dispatcher shutdown complete\n"); - println!("=== Demo Complete ==="); - println!("\nYou can view the emitted events in: {}", json_path); -} - -async fn simulate_build(bus: &BuildEventBus) { - println!("--- Simulating Build ---\n"); - - // 1. Build started - println!("1. Emitting BuildStarted event"); - bus.emit(BuildEvent::BuildStarted(BuildStarted { - invocation_id: "demo-invocation-123".to_string(), - command: "build //example:target".to_string(), - timestamp_millis: current_millis(), - working_directory: std::env::current_dir() - .unwrap() - .to_string_lossy() - .to_string(), - })); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - - // 2. Target started - println!("2. Emitting TargetStarted event"); - bus.emit(BuildEvent::TargetStarted(TargetStarted { - target_id: "target-1".to_string(), - label: "//example:target".to_string(), - target_kind: TargetKind::Binary, - timestamp_millis: current_millis(), - })); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - - // 3. Action started - compile - println!("3. Emitting ActionStarted event (compile)"); - bus.emit(BuildEvent::ActionStarted(ActionStarted { - action_id: "action-compile-1".to_string(), - action_name: "RustcCompile".to_string(), - target_id: "target-1".to_string(), - timestamp_millis: current_millis(), - })); - tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; - - // 4. Action completed - compile - println!("4. Emitting ActionCompleted event (compile)"); - bus.emit(BuildEvent::ActionCompleted(ActionCompleted { - action_id: "action-compile-1".to_string(), - success: true, - timestamp_millis: current_millis(), - exit_code: 0, - stdout: Some("Compiling example v0.1.0".to_string()), - stderr: None, - })); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - - // 5. Action started - link - println!("5. Emitting ActionStarted event (link)"); - bus.emit(BuildEvent::ActionStarted(ActionStarted { - action_id: "action-link-1".to_string(), - action_name: "RustcLink".to_string(), - target_id: "target-1".to_string(), - timestamp_millis: current_millis(), - })); - tokio::time::sleep(tokio::time::Duration::from_millis(150)).await; - - // 6. Action completed - link - println!("6. Emitting ActionCompleted event (link)"); - bus.emit(BuildEvent::ActionCompleted(ActionCompleted { - action_id: "action-link-1".to_string(), - success: true, - timestamp_millis: current_millis(), - exit_code: 0, - stdout: Some("Finished release [optimized] target(s)".to_string()), - stderr: None, - })); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - - // 7. Target completed - println!("7. Emitting TargetCompleted event"); - bus.emit(BuildEvent::TargetCompleted(TargetCompleted { - target_id: "target-1".to_string(), - label: "//example:target".to_string(), - success: true, - timestamp_millis: current_millis(), - error_message: None, - outputs: vec!["target/release/example".to_string()], - cache_hit: false, - })); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - - // 8. Build finished - println!("8. Emitting BuildFinished event"); - bus.emit(BuildEvent::BuildFinished(BuildFinished { - invocation_id: "demo-invocation-123".to_string(), - success: true, - timestamp_millis: current_millis(), - error_message: None, - })); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; -} diff --git a/deny.toml b/deny.toml index 00eedead47..985ef8d8c2 100644 --- a/deny.toml +++ b/deny.toml @@ -76,18 +76,16 @@ skip = [ { name = "vte", version = "0.11.1" }, - { name = "tower", version = "0.4.13" }, - { name = "tonic", version = "0.12.3" }, - { name = "socket2", version = "0.5.10" }, - { name = "rand_core", version = "0.6.4" }, - { name = "rand_chacha", version = "0.3.1" }, - { name = "rand", version = "0.8.5" }, - { name = "prost-types", version = "0.13.5" }, - { name = "prost-derive", version = "0.13.5" }, - { name = "prost", version = "0.13.5" }, { name = "indexmap", version = "1.9.3" }, - - + { name = "prost", version = "0.13.5" }, + { name = "prost-derive", version = "0.13.5" }, + { name = "prost-types", version = "0.13.5" }, + { name = "rand", version = "0.8.5" }, + { name = "rand_chacha", version = "0.3.1" }, + { name = "rand_core", version = "0.6.4" }, + { name = "socket2", version = "0.5.10" }, + { name = "tonic", version = "0.12.3" }, + { name = "tower", version = "0.4.13" }, ] skip-tree = [ diff --git a/flake.nix b/flake.nix index 11b032219b..933507b03c 100644 --- a/flake.nix +++ b/flake.nix @@ -18,11 +18,12 @@ buildInputs = with pkgs; [ cargo clippy - protobuf + protobuf rust-analyzer rustc rustfmt openssl + taplo ]; }; }; From 2da0eb2439380491aa7583e9f723f5df21edc2f3 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Tue, 14 Oct 2025 13:12:24 -0700 Subject: [PATCH 14/39] Reuse canonical proto types --- Cargo.lock | 3 +- crates/build-events-proto/src/convert.rs | 390 ------------------ crates/build-events-proto/src/lib.rs | 28 +- .../src/spongebob_subscriber_v2.rs | 8 +- crates/build-events-proto/src/subscriber.rs | 32 +- crates/build-events/Cargo.toml | 9 +- crates/build-events/src/bus.rs | 104 +++-- crates/build-events/src/dispatcher.rs | 41 +- crates/build-events/src/events.rs | 176 ++------ crates/build-events/src/lib.rs | 21 +- crates/build-events/src/subscriber.rs | 15 +- .../build-events/src/subscribers/json_file.rs | 178 ++++++-- crates/build-events/src/subscribers/logger.rs | 62 +-- crates/build-events/tests/integration_test.rs | 169 +++++--- crates/build-sandbox/Cargo.toml | 6 +- crates/build-sandbox/src/executor.rs | 6 +- crates/minimal/src/cmd_build.rs | 63 ++- 17 files changed, 496 insertions(+), 815 deletions(-) delete mode 100644 crates/build-events-proto/src/convert.rs diff --git a/Cargo.lock b/Cargo.lock index 6195623cc9..db9e63a802 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -346,6 +346,7 @@ name = "build-events" version = "0.0.1" dependencies = [ "async-trait", + "minimal_spongebob_community_neoeinstein-prost", "serde", "serde_json", "tempfile", @@ -377,10 +378,10 @@ dependencies = [ name = "build-sandbox" version = "0.1.0" dependencies = [ + "build-events", "common", "globset", "hakoniwa", - "minimal_spongebob_community_neoeinstein-prost", "nickel-lang-core", "serde", "spongebob", diff --git a/crates/build-events-proto/src/convert.rs b/crates/build-events-proto/src/convert.rs deleted file mode 100644 index 7d4e61265c..0000000000 --- a/crates/build-events-proto/src/convert.rs +++ /dev/null @@ -1,390 +0,0 @@ -//! Type conversions between Rust and protobuf types -//! -//! This module provides conversion functions between the pure Rust `build_events` -//! types and the BSR-generated canonical protobuf types from spongebob.v1. -//! -//! # Extension Trait -//! -//! For ergonomic conversion, use the [`ToProto`] trait which provides a `.to_proto()` method: -//! -//! ``` -//! use build_events::events::{BuildStarted, BuildEvent}; -//! use build_events_proto::convert::ToProto; -//! -//! let event = BuildEvent::BuildStarted(BuildStarted { -//! invocation_id: "test-123".to_string(), -//! command: "build".to_string(), -//! timestamp_millis: 123456789, -//! working_directory: "/tmp".to_string(), -//! }); -//! -//! let proto_event = event.to_proto("test-123"); -//! ``` - -use crate::proto; -use build_events::events; - -// ============================================================================ -// Rust -> Proto conversions -// ============================================================================ - -/// Convert Rust BuildEvent to proto BuildEvent with invocation_id at wrapper level -/// -/// The canonical spongebob proto has `invocation_id` both at the BuildEvent wrapper level -/// and inside each event type. This function ensures the wrapper-level invocation_id is set. -pub fn to_proto_build_event(event: &events::BuildEvent, invocation_id: &str) -> proto::BuildEvent { - let proto_event = match event { - events::BuildEvent::BuildStarted(e) => { - proto::build_event::Event::BuildStarted(to_proto_build_started(e)) - } - events::BuildEvent::BuildFinished(e) => { - proto::build_event::Event::BuildFinished(to_proto_build_finished(e)) - } - events::BuildEvent::TargetStarted(e) => { - proto::build_event::Event::TargetStarted(to_proto_target_started(e)) - } - events::BuildEvent::TargetCompleted(e) => { - proto::build_event::Event::TargetCompleted(to_proto_target_completed(e)) - } - events::BuildEvent::ActionStarted(e) => { - proto::build_event::Event::ActionStarted(to_proto_action_started(e)) - } - events::BuildEvent::ActionCompleted(e) => { - proto::build_event::Event::ActionCompleted(to_proto_action_completed(e)) - } - events::BuildEvent::BuildMetadata(e) => { - proto::build_event::Event::BuildMetadata(to_proto_build_metadata(e)) - } - }; - - proto::BuildEvent { - invocation_id: invocation_id.to_string(), - event: Some(proto_event), - } -} - -pub fn to_proto_build_started(event: &events::BuildStarted) -> proto::BuildStarted { - proto::BuildStarted { - invocation_id: event.invocation_id.clone(), - command: event.command.clone(), - timestamp_millis: event.timestamp_millis, - working_directory: event.working_directory.clone(), - } -} - -pub fn to_proto_build_finished(event: &events::BuildFinished) -> proto::BuildFinished { - proto::BuildFinished { - invocation_id: event.invocation_id.clone(), - success: event.success, - timestamp_millis: event.timestamp_millis, - error_message: event.error_message.clone(), - } -} - -pub fn to_proto_target_started(event: &events::TargetStarted) -> proto::TargetStarted { - proto::TargetStarted { - target_id: event.target_id.clone(), - label: event.label.clone(), - target_kind: to_proto_target_kind(&event.target_kind) as i32, - timestamp_millis: event.timestamp_millis, - } -} - -pub fn to_proto_target_completed(event: &events::TargetCompleted) -> proto::TargetCompleted { - proto::TargetCompleted { - target_id: event.target_id.clone(), - label: event.label.clone(), - success: event.success, - timestamp_millis: event.timestamp_millis, - error_message: event.error_message.clone(), - outputs: event.outputs.clone(), - cache_hit: event.cache_hit, - } -} - -pub fn to_proto_action_started(event: &events::ActionStarted) -> proto::ActionStarted { - proto::ActionStarted { - action_id: event.action_id.clone(), - action_name: event.action_name.clone(), - target_id: event.target_id.clone(), - timestamp_millis: event.timestamp_millis, - } -} - -pub fn to_proto_action_completed(event: &events::ActionCompleted) -> proto::ActionCompleted { - proto::ActionCompleted { - action_id: event.action_id.clone(), - success: event.success, - timestamp_millis: event.timestamp_millis, - exit_code: event.exit_code, - stdout: event.stdout.clone(), - stderr: event.stderr.clone(), - } -} - -pub fn to_proto_build_metadata(event: &events::BuildMetadata) -> proto::BuildMetadata { - proto::BuildMetadata { - metadata: event.metadata.clone(), - } -} - -pub fn to_proto_target_kind(kind: &events::TargetKind) -> proto::TargetKind { - match kind { - events::TargetKind::Binary => proto::TargetKind::Binary, - events::TargetKind::Library => proto::TargetKind::Library, - events::TargetKind::Test => proto::TargetKind::Test, - } -} - -// ============================================================================ -// Proto -> Rust conversions -// ============================================================================ - -pub fn from_proto_build_event( - event: proto::BuildEvent, -) -> Result { - let event = event.event.ok_or(ConversionError::MissingField("event"))?; - - match event { - proto::build_event::Event::BuildStarted(e) => Ok(events::BuildEvent::BuildStarted( - from_proto_build_started(e)?, - )), - proto::build_event::Event::BuildFinished(e) => Ok(events::BuildEvent::BuildFinished( - from_proto_build_finished(e)?, - )), - proto::build_event::Event::TargetStarted(e) => Ok(events::BuildEvent::TargetStarted( - from_proto_target_started(e)?, - )), - proto::build_event::Event::TargetCompleted(e) => Ok(events::BuildEvent::TargetCompleted( - from_proto_target_completed(e)?, - )), - proto::build_event::Event::ActionStarted(e) => Ok(events::BuildEvent::ActionStarted( - from_proto_action_started(e)?, - )), - proto::build_event::Event::ActionCompleted(e) => Ok(events::BuildEvent::ActionCompleted( - from_proto_action_completed(e)?, - )), - proto::build_event::Event::BuildMetadata(e) => Ok(events::BuildEvent::BuildMetadata( - from_proto_build_metadata(e)?, - )), - } -} - -pub fn from_proto_build_started( - event: proto::BuildStarted, -) -> Result { - Ok(events::BuildStarted { - invocation_id: event.invocation_id, - command: event.command, - timestamp_millis: event.timestamp_millis, - working_directory: event.working_directory, - }) -} - -pub fn from_proto_build_finished( - event: proto::BuildFinished, -) -> Result { - Ok(events::BuildFinished { - invocation_id: event.invocation_id, - success: event.success, - timestamp_millis: event.timestamp_millis, - error_message: event.error_message, - }) -} - -pub fn from_proto_target_started( - event: proto::TargetStarted, -) -> Result { - let target_kind = proto::TargetKind::try_from(event.target_kind) - .map_err(|_| ConversionError::InvalidEnum("target_kind", event.target_kind))?; - - Ok(events::TargetStarted { - target_id: event.target_id, - label: event.label, - target_kind: from_proto_target_kind(target_kind)?, - timestamp_millis: event.timestamp_millis, - }) -} - -pub fn from_proto_target_completed( - event: proto::TargetCompleted, -) -> Result { - Ok(events::TargetCompleted { - target_id: event.target_id, - label: event.label, - success: event.success, - timestamp_millis: event.timestamp_millis, - error_message: event.error_message, - outputs: event.outputs, - cache_hit: event.cache_hit, - }) -} - -pub fn from_proto_action_started( - event: proto::ActionStarted, -) -> Result { - Ok(events::ActionStarted { - action_id: event.action_id, - action_name: event.action_name, - target_id: event.target_id, - timestamp_millis: event.timestamp_millis, - }) -} - -pub fn from_proto_action_completed( - event: proto::ActionCompleted, -) -> Result { - Ok(events::ActionCompleted { - action_id: event.action_id, - success: event.success, - timestamp_millis: event.timestamp_millis, - exit_code: event.exit_code, - stdout: event.stdout, - stderr: event.stderr, - }) -} - -pub fn from_proto_build_metadata( - event: proto::BuildMetadata, -) -> Result { - Ok(events::BuildMetadata { - metadata: event.metadata, - }) -} - -pub fn from_proto_target_kind( - kind: proto::TargetKind, -) -> Result { - match kind { - proto::TargetKind::Binary => Ok(events::TargetKind::Binary), - proto::TargetKind::Library => Ok(events::TargetKind::Library), - proto::TargetKind::Test => Ok(events::TargetKind::Test), - proto::TargetKind::Unspecified => Err(ConversionError::InvalidEnum("target_kind", 0)), - } -} - -// ============================================================================ -// Extension Trait for Ergonomic Conversion -// ============================================================================ - -/// Extension trait providing ergonomic `.to_proto()` method for build events -/// -/// This trait provides a convenient way to convert Rust build events to protobuf -/// format with method syntax instead of calling standalone functions. -/// -/// # Example -/// -/// ``` -/// use build_events::events::{BuildStarted, BuildEvent, current_millis}; -/// use build_events_proto::convert::ToProto; -/// -/// let event = BuildEvent::BuildStarted(BuildStarted { -/// invocation_id: "test-123".to_string(), -/// command: "build".to_string(), -/// timestamp_millis: current_millis(), -/// working_directory: "/tmp".to_string(), -/// }); -/// -/// // Ergonomic method syntax -/// let proto_event = event.to_proto("test-123"); -/// ``` -pub trait ToProto { - /// Convert this event to a protobuf BuildEvent with the specified invocation_id - /// - /// The invocation_id is required because the canonical spongebob proto has - /// `invocation_id` at the BuildEvent wrapper level, separate from the inner - /// event types. - fn to_proto(&self, invocation_id: &str) -> proto::BuildEvent; -} - -impl ToProto for events::BuildEvent { - fn to_proto(&self, invocation_id: &str) -> proto::BuildEvent { - to_proto_build_event(self, invocation_id) - } -} - -// ============================================================================ -// Error types -// ============================================================================ - -/// Error that can occur during type conversion -#[derive(Debug, thiserror::Error)] -pub enum ConversionError { - #[error("Missing required field: {0}")] - MissingField(&'static str), - - #[error("Invalid enum value for {0}: {1}")] - InvalidEnum(&'static str, i32), -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_build_started_round_trip() { - let rust_event = events::BuildStarted { - invocation_id: "test-123".to_string(), - command: "build".to_string(), - timestamp_millis: 1234567890, - working_directory: "/tmp".to_string(), - }; - - let proto_event = to_proto_build_started(&rust_event); - let converted_back = from_proto_build_started(proto_event).unwrap(); - - assert_eq!(rust_event, converted_back); - } - - #[test] - fn test_build_event_round_trip() { - let rust_event = events::BuildEvent::BuildStarted(events::BuildStarted { - invocation_id: "test-456".to_string(), - command: "test".to_string(), - timestamp_millis: 9876543210, - working_directory: "/home/user".to_string(), - }); - - let proto_event = to_proto_build_event(&rust_event, "test-456"); - let converted_back = from_proto_build_event(proto_event).unwrap(); - - assert_eq!(rust_event, converted_back); - } - - #[test] - fn test_target_kind_conversion() { - let rust_kind = events::TargetKind::Binary; - let proto_kind = to_proto_target_kind(&rust_kind); - let converted_back = from_proto_target_kind(proto_kind).unwrap(); - - assert_eq!(rust_kind, converted_back); - } - - #[test] - fn test_optional_fields() { - let rust_event = events::BuildFinished { - invocation_id: "test-789".to_string(), - success: false, - timestamp_millis: 1111111111, - error_message: Some("build failed".to_string()), - }; - - let proto_event = to_proto_build_finished(&rust_event); - let converted_back = from_proto_build_finished(proto_event).unwrap(); - - assert_eq!(rust_event, converted_back); - - // Test with None - let rust_event_none = events::BuildFinished { - invocation_id: "test-000".to_string(), - success: true, - timestamp_millis: 2222222222, - error_message: None, - }; - - let proto_event_none = to_proto_build_finished(&rust_event_none); - let converted_back_none = from_proto_build_finished(proto_event_none).unwrap(); - - assert_eq!(rust_event_none, converted_back_none); - } -} diff --git a/crates/build-events-proto/src/lib.rs b/crates/build-events-proto/src/lib.rs index ff802fab31..31009fe4b7 100644 --- a/crates/build-events-proto/src/lib.rs +++ b/crates/build-events-proto/src/lib.rs @@ -89,24 +89,24 @@ //! } //! ``` //! -//! # Type Conversions +//! # Type Usage //! -//! Convert between Rust and protobuf types: +//! Since build-events now uses proto types directly, no conversion is needed: //! //! ``` -//! use build_events::events::{BuildStarted, current_millis}; -//! use build_events::BuildEvent; -//! use build_events_proto::{proto, convert::ToProto}; +//! use build_events::events::{build_event, BuildEvent, BuildStarted, current_millis}; +//! use build_events_proto::proto; //! -//! // Rust -> Proto -//! let rust_event = BuildEvent::BuildStarted(BuildStarted { +//! // Create proto event (build-events types ARE proto types) +//! let event = BuildEvent { //! invocation_id: "test-123".to_string(), -//! command: "build".to_string(), -//! timestamp_millis: current_millis(), -//! working_directory: "/tmp".to_string(), -//! }); -//! -//! let proto_event: proto::BuildEvent = rust_event.to_proto("test-123"); +//! event: Some(build_event::Event::BuildStarted(BuildStarted { +//! invocation_id: "test-123".to_string(), +//! command: "build".to_string(), +//! timestamp_millis: current_millis(), +//! working_directory: "/tmp".to_string(), +//! })), +//! }; //! ``` // Use BSR-generated protobuf code from canonical spongebob.v1 package @@ -120,7 +120,6 @@ pub mod proto { }; } -pub mod convert; pub mod service; pub mod subscriber; @@ -128,7 +127,6 @@ pub mod subscriber; pub mod spongebob_subscriber_v2; // Re-export main types for convenience -pub use convert::{ConversionError, ToProto, to_proto_build_event}; pub use service::BuildEventServiceImpl; pub use subscriber::GrpcStreamSubscriber; diff --git a/crates/build-events-proto/src/spongebob_subscriber_v2.rs b/crates/build-events-proto/src/spongebob_subscriber_v2.rs index 6765353d08..88d0ca4863 100644 --- a/crates/build-events-proto/src/spongebob_subscriber_v2.rs +++ b/crates/build-events-proto/src/spongebob_subscriber_v2.rs @@ -7,9 +7,6 @@ use build_events::{BuildEvent, BuildEventSubscriber, SubscriberError}; #[cfg(feature = "spongebob-subscriber")] use tracing::warn; -#[cfg(feature = "spongebob-subscriber")] -use crate::convert::ToProto; - /// Subscriber that publishes build events to Spongebob service /// /// This subscriber uses the workspace `spongebob` crate to connect to the @@ -71,11 +68,8 @@ impl BuildEventSubscriber for SpongeBobSubscriberV2 { // Get lock on invocation let mut invocation = self.invocation.lock().await; - // Convert Rust event to proto (need invocation_id from the invocation) - let proto_event = event.to_proto(invocation.invocation_id()); - // Publish event - match invocation.publish_build_event(proto_event).await { + match invocation.publish_build_event(event.clone()).await { Ok(()) => { tracing::debug!("Successfully published build event to Spongebob"); Ok(()) diff --git a/crates/build-events-proto/src/subscriber.rs b/crates/build-events-proto/src/subscriber.rs index 463361b81b..cb14831565 100644 --- a/crates/build-events-proto/src/subscriber.rs +++ b/crates/build-events-proto/src/subscriber.rs @@ -89,21 +89,6 @@ impl GrpcStreamSubscriber { #[async_trait] impl BuildEventSubscriber for GrpcStreamSubscriber { async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { - // Extract invocation_id from the event (all variants have it) - let invocation_id = match event { - BuildEvent::BuildStarted(e) => &e.invocation_id, - BuildEvent::BuildFinished(e) => &e.invocation_id, - BuildEvent::TargetStarted(_) => "", // Targets don't have invocation_id in inner fields - BuildEvent::TargetCompleted(_) => "", - BuildEvent::ActionStarted(_) => "", - BuildEvent::ActionCompleted(_) => "", - BuildEvent::BuildMetadata(_) => "", // Metadata doesn't have invocation_id in inner fields - }; - - // Convert Rust event to proto using ergonomic extension trait - use crate::convert::ToProto; - let proto_event = event.to_proto(invocation_id); - // Broadcast to all subscribers // If there are no subscribers, this is a no-op if self.sender.receiver_count() == 0 { @@ -111,7 +96,7 @@ impl BuildEventSubscriber for GrpcStreamSubscriber { } // Send to broadcast channel (will drop if no receivers) - if let Err(e) = self.sender.send(proto_event) { + if let Err(e) = self.sender.send(event.clone()) { // If channel has no receivers, this is expected warn!("GrpcStreamSubscriber: no active receivers ({})", e); } @@ -137,15 +122,20 @@ mod tests { #[tokio::test] async fn test_subscriber_stream() { + use build_events::events::build_event; + let subscriber = GrpcStreamSubscriber::new(10); let mut stream = subscriber.subscribe_stream(); - let event = BuildEvent::BuildStarted(BuildStarted { + let event = BuildEvent { invocation_id: "test-123".to_string(), - command: "test".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - }); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command: "test".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }; // Send event subscriber.on_event(&event).await.unwrap(); diff --git a/crates/build-events/Cargo.toml b/crates/build-events/Cargo.toml index d74c62de97..76ea6e0af6 100644 --- a/crates/build-events/Cargo.toml +++ b/crates/build-events/Cargo.toml @@ -7,12 +7,15 @@ license = "MIT OR Apache-2.0" publish.workspace = true [dependencies] -tokio.workspace = true -serde.workspace = true +async-trait = "0.1" serde_json.workspace = true +serde.workspace = true thiserror.workspace = true +tokio.workspace = true tracing.workspace = true -async-trait = "0.1" + +# Proto types as single source of truth +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251010205210-42ae8fd1517e.1", registry = "buf" } [dev-dependencies] serde_json.workspace = true diff --git a/crates/build-events/src/bus.rs b/crates/build-events/src/bus.rs index aa0ba57606..856216d6ba 100644 --- a/crates/build-events/src/bus.rs +++ b/crates/build-events/src/bus.rs @@ -22,17 +22,19 @@ use tokio::sync::broadcast; /// let bus = BuildEventBus::new(1000); /// let mut receiver = bus.subscribe(); /// -/// bus.emit(BuildEvent::BuildStarted(BuildStarted { +/// bus.emit(BuildEvent { /// invocation_id: "test-123".to_string(), -/// command: "build".to_string(), -/// timestamp_millis: current_millis(), -/// working_directory: "/tmp".to_string(), -/// })); +/// event: Some(build_event::Event::BuildStarted(BuildStarted { +/// invocation_id: "test-123".to_string(), +/// command: "build".to_string(), +/// timestamp_millis: current_millis(), +/// working_directory: "/tmp".to_string(), +/// })), +/// }); /// /// let event = receiver.recv().await.unwrap(); -/// match event { -/// BuildEvent::BuildStarted(_) => println!("Build started!"), -/// _ => {} +/// if let Some(build_event::Event::BuildStarted(_)) = event.event { +/// println!("Build started!"); /// } /// } /// ``` @@ -77,12 +79,15 @@ impl BuildEventBus { /// use build_events::{BuildEventBus, BuildEvent, events::*}; /// /// let bus = BuildEventBus::new(1000); - /// bus.emit(BuildEvent::BuildStarted(BuildStarted { + /// bus.emit(BuildEvent { /// invocation_id: "test-123".to_string(), - /// command: "build".to_string(), - /// timestamp_millis: current_millis(), - /// working_directory: "/tmp".to_string(), - /// })); + /// event: Some(build_event::Event::BuildStarted(BuildStarted { + /// invocation_id: "test-123".to_string(), + /// command: "build".to_string(), + /// timestamp_millis: current_millis(), + /// working_directory: "/tmp".to_string(), + /// })), + /// }); /// ``` pub fn emit(&self, event: BuildEvent) { // Use try_send to avoid blocking. If the channel is full, the event @@ -133,15 +138,20 @@ mod tests { #[tokio::test] async fn test_bus_emit_and_receive() { + use crate::events::build_event; + let bus = BuildEventBus::new(100); let mut receiver = bus.subscribe(); - let event = BuildEvent::BuildStarted(BuildStarted { + let event = BuildEvent { invocation_id: "test-123".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - }); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }; bus.emit(event.clone()); @@ -151,16 +161,21 @@ mod tests { #[tokio::test] async fn test_bus_multiple_subscribers() { + use crate::events::build_event; + let bus = BuildEventBus::new(100); let mut receiver1 = bus.subscribe(); let mut receiver2 = bus.subscribe(); - let event = BuildEvent::BuildStarted(BuildStarted { + let event = BuildEvent { invocation_id: "test-456".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - }); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test-456".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }; bus.emit(event.clone()); @@ -172,16 +187,21 @@ mod tests { #[tokio::test] async fn test_bus_clone() { + use crate::events::build_event; + let bus = BuildEventBus::new(100); let bus_clone = bus.clone(); let mut receiver = bus.subscribe(); - let event = BuildEvent::BuildStarted(BuildStarted { + let event = BuildEvent { invocation_id: "test-789".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - }); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test-789".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }; // Emit from the cloned bus bus_clone.emit(event.clone()); @@ -204,14 +224,19 @@ mod tests { #[tokio::test] async fn test_late_subscription() { + use crate::events::build_event; + let bus = BuildEventBus::new(100); - let event1 = BuildEvent::BuildStarted(BuildStarted { + let event1 = BuildEvent { invocation_id: "early".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - }); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "early".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }; // Emit before subscription bus.emit(event1.clone()); @@ -219,12 +244,15 @@ mod tests { // Subscribe after event was emitted let mut receiver = bus.subscribe(); - let event2 = BuildEvent::BuildStarted(BuildStarted { + let event2 = BuildEvent { invocation_id: "late".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - }); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "late".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }; bus.emit(event2.clone()); diff --git a/crates/build-events/src/dispatcher.rs b/crates/build-events/src/dispatcher.rs index 6e62b4366c..7b5b16c2ea 100644 --- a/crates/build-events/src/dispatcher.rs +++ b/crates/build-events/src/dispatcher.rs @@ -148,7 +148,7 @@ impl BuildEventDispatcher { #[cfg(test)] mod tests { use super::*; - use crate::events::{BuildStarted, current_millis}; + use crate::events::{build_event, BuildStarted, current_millis}; use crate::subscriber::SubscriberError; use async_trait::async_trait; use std::sync::Arc; @@ -190,12 +190,15 @@ mod tests { // Send some events for i in 0..5 { sender - .send(BuildEvent::BuildStarted(BuildStarted { + .send(BuildEvent { invocation_id: format!("test-{}", i), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - })) + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: format!("test-{}", i), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }) .unwrap(); } @@ -234,12 +237,15 @@ mod tests { // Send events for i in 0..3 { sender - .send(BuildEvent::BuildStarted(BuildStarted { + .send(BuildEvent { invocation_id: format!("test-{}", i), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - })) + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: format!("test-{}", i), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }) .unwrap(); } @@ -288,12 +294,15 @@ mod tests { // Send event sender - .send(BuildEvent::BuildStarted(BuildStarted { + .send(BuildEvent { invocation_id: "test".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - })) + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }) .unwrap(); tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; diff --git a/crates/build-events/src/events.rs b/crates/build-events/src/events.rs index b61d79741c..59519e2601 100644 --- a/crates/build-events/src/events.rs +++ b/crates/build-events/src/events.rs @@ -1,139 +1,14 @@ //! Build event type definitions //! -//! This module defines the core event types emitted during a build. -//! Events are strongly-typed Rust enums and structs with serde support -//! for easy serialization to JSON, protobuf, or other formats. +//! This module re-exports protobuf event types from the canonical spongebob.v1 schema +//! as the single source of truth for build events. Proto types are used directly +//! throughout the build-events system for consistency and to eliminate duplication. -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -/// Main build event enum that wraps all possible event types -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum BuildEvent { - /// Build process started - BuildStarted(BuildStarted), - /// Build process finished - BuildFinished(BuildFinished), - /// Target started building - TargetStarted(TargetStarted), - /// Target completed building - TargetCompleted(TargetCompleted), - /// Action started executing - ActionStarted(ActionStarted), - /// Action completed executing - ActionCompleted(ActionCompleted), - /// Build metadata (user, branch, commit, etc.) - BuildMetadata(BuildMetadata), -} - -/// Event emitted when a build starts -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub struct BuildStarted { - /// Unique identifier for this build invocation - pub invocation_id: String, - /// Command string for display - pub command: String, - /// Unix timestamp in milliseconds - pub timestamp_millis: i64, - /// Working directory where build was started - pub working_directory: String, -} - -/// Event emitted when a build finishes -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub struct BuildFinished { - /// Unique identifier for this build invocation - pub invocation_id: String, - /// Whether the build succeeded - pub success: bool, - /// Unix timestamp in milliseconds - pub timestamp_millis: i64, - /// Optional error message if build failed - pub error_message: Option, -} - -/// Event emitted when a target starts building -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub struct TargetStarted { - /// Unique identifier for this target - pub target_id: String, - /// Target label (e.g., "//foo:bar") - pub label: String, - /// Type of target being built - pub target_kind: TargetKind, - /// Unix timestamp in milliseconds - pub timestamp_millis: i64, -} - -/// Event emitted when a target completes -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub struct TargetCompleted { - /// Unique identifier for this target (must match TargetStarted.target_id) - pub target_id: String, - /// Target label (e.g., "//foo:bar") - pub label: String, - /// Whether the target built successfully - pub success: bool, - /// Unix timestamp in milliseconds - pub timestamp_millis: i64, - /// Optional error message if target failed - pub error_message: Option, - /// Paths to generated output files - pub outputs: Vec, - /// Whether this target was served from cache - pub cache_hit: bool, -} - -/// Event emitted when an action starts executing -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub struct ActionStarted { - /// Unique identifier for this action - pub action_id: String, - /// Human-readable name for the action (e.g., "CppCompile", "Link") - pub action_name: String, - /// Target this action belongs to (if any) - pub target_id: String, - /// Unix timestamp in milliseconds - pub timestamp_millis: i64, -} - -/// Event emitted when an action completes -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub struct ActionCompleted { - /// Unique identifier for this action - pub action_id: String, - /// Whether the action succeeded - pub success: bool, - /// Unix timestamp in milliseconds - pub timestamp_millis: i64, - /// Exit code of the action - pub exit_code: i32, - /// Optional stdout from the action - pub stdout: Option, - /// Optional stderr from the action - pub stderr: Option, -} - -/// Event containing arbitrary key-value metadata about the build environment -/// Standard keys: "user", "branch", "commit", "repo_url" -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -pub struct BuildMetadata { - /// Metadata key-value pairs - pub metadata: HashMap, -} - -/// Type of build target -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "snake_case")] -pub enum TargetKind { - /// Binary executable target - Binary, - /// Library target - Library, - /// Test target - Test, -} +// Re-export proto types as the canonical event types +pub use minimal_spongebob_community_neoeinstein_prost::spongebob::v1::{ + build_event, ActionCompleted, ActionStarted, BuildEvent, BuildFinished, BuildMetadata, + BuildStarted, TargetCompleted, TargetKind, TargetStarted, +}; /// Helper function to get current timestamp in milliseconds /// @@ -164,23 +39,30 @@ mod tests { } #[test] - fn test_event_serialization() { - let event = BuildEvent::BuildStarted(BuildStarted { + fn test_event_construction() { + let event = BuildEvent { invocation_id: "test-123".to_string(), - command: "build".to_string(), - timestamp_millis: 1234567890, - working_directory: "/tmp".to_string(), - }); - - let json = serde_json::to_string(&event).unwrap(); - let deserialized: BuildEvent = serde_json::from_str(&json).unwrap(); - assert_eq!(event, deserialized); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command: "build".to_string(), + timestamp_millis: 1234567890, + working_directory: "/tmp".to_string(), + })), + }; + + assert_eq!(event.invocation_id, "test-123"); + assert!(matches!( + event.event, + Some(build_event::Event::BuildStarted(_)) + )); } #[test] - fn test_target_kind_serialization() { - let kind = TargetKind::Binary; - let json = serde_json::to_string(&kind).unwrap(); - assert_eq!(json, "\"binary\""); + fn test_target_kind_values() { + // Verify proto enum values + assert_eq!(TargetKind::Unspecified as i32, 0); + assert_eq!(TargetKind::Binary as i32, 1); + assert_eq!(TargetKind::Library as i32, 2); + assert_eq!(TargetKind::Test as i32, 3); } } diff --git a/crates/build-events/src/lib.rs b/crates/build-events/src/lib.rs index cdcb1606b3..685884958d 100644 --- a/crates/build-events/src/lib.rs +++ b/crates/build-events/src/lib.rs @@ -27,7 +27,7 @@ //! ```no_run //! use build_events::{ //! BuildEventBus, BuildEventDispatcher, BuildEvent, -//! events::{BuildStarted, current_millis}, +//! events::{build_event, BuildStarted, current_millis}, //! subscribers::{LoggerSubscriber, JsonFileWriter}, //! }; //! @@ -52,15 +52,18 @@ //! }); //! //! // Emit events from build -//! event_bus.emit(BuildEvent::BuildStarted(BuildStarted { +//! event_bus.emit(BuildEvent { //! invocation_id: "build-123".to_string(), -//! command: "cargo build".to_string(), -//! timestamp_millis: current_millis(), -//! working_directory: std::env::current_dir() -//! .unwrap() -//! .to_string_lossy() -//! .to_string(), -//! })); +//! event: Some(build_event::Event::BuildStarted(BuildStarted { +//! invocation_id: "build-123".to_string(), +//! command: "cargo build".to_string(), +//! timestamp_millis: current_millis(), +//! working_directory: std::env::current_dir() +//! .unwrap() +//! .to_string_lossy() +//! .to_string(), +//! })), +//! }); //! //! // ... perform build ... //! diff --git a/crates/build-events/src/subscriber.rs b/crates/build-events/src/subscriber.rs index c53e7f00d6..f1a4c705c3 100644 --- a/crates/build-events/src/subscriber.rs +++ b/crates/build-events/src/subscriber.rs @@ -85,7 +85,7 @@ pub trait BuildEventSubscriber: Send + Sync { #[cfg(test)] mod tests { use super::*; - use crate::events::{BuildStarted, current_millis}; + use crate::events::{build_event, BuildStarted, current_millis}; struct TestSubscriber { name: String, @@ -110,12 +110,15 @@ mod tests { assert_eq!(subscriber.name(), "test"); - let event = BuildEvent::BuildStarted(BuildStarted { + let event = BuildEvent { invocation_id: "test-123".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - }); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }; let result = subscriber.on_event(&event).await; assert!(result.is_ok()); diff --git a/crates/build-events/src/subscribers/json_file.rs b/crates/build-events/src/subscribers/json_file.rs index dc0e085633..38a396e86d 100644 --- a/crates/build-events/src/subscribers/json_file.rs +++ b/crates/build-events/src/subscribers/json_file.rs @@ -2,7 +2,7 @@ //! //! Writes build events to a file in JSON-lines format (one JSON object per line). -use crate::events::BuildEvent; +use crate::events::{build_event, BuildEvent, TargetKind}; use crate::subscriber::{BuildEventSubscriber, SubscriberError}; use async_trait::async_trait; use std::io::Write; @@ -67,15 +67,117 @@ impl std::fmt::Debug for JsonFileWriter { } } +/// Convert proto BuildEvent to JSON value (custom serialization since proto types don't have serde) +fn event_to_json(event: &BuildEvent) -> serde_json::Value { + let event_type_json = match &event.event { + Some(build_event::Event::BuildStarted(e)) => { + serde_json::json!({ + "type": "build_started", + "build_started": { + "invocation_id": e.invocation_id, + "command": e.command, + "timestamp_millis": e.timestamp_millis, + "working_directory": e.working_directory, + } + }) + } + Some(build_event::Event::BuildFinished(e)) => { + serde_json::json!({ + "type": "build_finished", + "build_finished": { + "invocation_id": e.invocation_id, + "success": e.success, + "timestamp_millis": e.timestamp_millis, + "error_message": e.error_message, + } + }) + } + Some(build_event::Event::TargetStarted(e)) => { + let target_kind = match TargetKind::try_from(e.target_kind) { + Ok(TargetKind::Binary) => "binary", + Ok(TargetKind::Library) => "library", + Ok(TargetKind::Test) => "test", + _ => "unspecified", + }; + serde_json::json!({ + "type": "target_started", + "target_started": { + "target_id": e.target_id, + "label": e.label, + "target_kind": target_kind, + "timestamp_millis": e.timestamp_millis, + } + }) + } + Some(build_event::Event::TargetCompleted(e)) => { + serde_json::json!({ + "type": "target_completed", + "target_completed": { + "target_id": e.target_id, + "label": e.label, + "success": e.success, + "timestamp_millis": e.timestamp_millis, + "error_message": e.error_message, + "outputs": e.outputs, + "cache_hit": e.cache_hit, + } + }) + } + Some(build_event::Event::ActionStarted(e)) => { + serde_json::json!({ + "type": "action_started", + "action_started": { + "action_id": e.action_id, + "action_name": e.action_name, + "target_id": e.target_id, + "timestamp_millis": e.timestamp_millis, + } + }) + } + Some(build_event::Event::ActionCompleted(e)) => { + serde_json::json!({ + "type": "action_completed", + "action_completed": { + "action_id": e.action_id, + "success": e.success, + "timestamp_millis": e.timestamp_millis, + "exit_code": e.exit_code, + "stdout": e.stdout, + "stderr": e.stderr, + } + }) + } + Some(build_event::Event::BuildMetadata(e)) => { + serde_json::json!({ + "type": "build_metadata", + "build_metadata": { + "metadata": e.metadata, + } + }) + } + None => { + serde_json::json!({ + "type": "unknown", + }) + } + }; + + serde_json::json!({ + "invocation_id": event.invocation_id, + "event": event_type_json, + }) +} + #[async_trait] impl BuildEventSubscriber for JsonFileWriter { async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { - let json = serde_json::to_string(event).map_err(|e| { + let json = event_to_json(event); + let json_string = serde_json::to_string(&json).map_err(|e| { SubscriberError::Serialization(format!("Failed to serialize event: {}", e)) })?; let mut file = self.file.lock().unwrap(); - writeln!(file, "{}", json)?; + writeln!(file, "{}", json_string)?; file.flush()?; Ok(()) @@ -95,7 +197,7 @@ impl BuildEventSubscriber for JsonFileWriter { #[cfg(test)] mod tests { use super::*; - use crate::events::{BuildStarted, TargetKind, TargetStarted, current_millis}; + use crate::events::{BuildStarted, TargetStarted, current_millis}; use std::io::{BufRead, BufReader}; #[tokio::test] @@ -107,12 +209,15 @@ mod tests { assert_eq!(writer.name(), "JsonFileWriter"); assert_eq!(writer.path(), file_path.as_path()); - let event = BuildEvent::BuildStarted(BuildStarted { + let event = BuildEvent { invocation_id: "test-123".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - }); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }; let result = writer.on_event(&event).await; assert!(result.is_ok()); @@ -128,8 +233,9 @@ mod tests { assert_eq!(lines.len(), 1); - let deserialized: BuildEvent = serde_json::from_str(&lines[0]).unwrap(); - assert_eq!(deserialized, event); + let parsed: serde_json::Value = serde_json::from_str(&lines[0]).unwrap(); + assert_eq!(parsed["invocation_id"], "test-123"); + assert_eq!(parsed["event"]["type"], "build_started"); } #[tokio::test] @@ -140,18 +246,24 @@ mod tests { let writer = JsonFileWriter::new(&file_path).unwrap(); let events = vec![ - BuildEvent::BuildStarted(BuildStarted { + BuildEvent { invocation_id: "test-1".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - }), - BuildEvent::TargetStarted(TargetStarted { - target_id: "target-1".to_string(), - label: "//foo:bar".to_string(), - target_kind: TargetKind::Binary, - timestamp_millis: current_millis(), - }), + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test-1".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }, + BuildEvent { + invocation_id: "test-1".to_string(), + event: Some(build_event::Event::TargetStarted(TargetStarted { + target_id: "target-1".to_string(), + label: "//foo:bar".to_string(), + target_kind: TargetKind::Binary as i32, + timestamp_millis: current_millis(), + })), + }, ]; for event in &events { @@ -168,10 +280,11 @@ mod tests { assert_eq!(lines.len(), 2); - for (i, line) in lines.iter().enumerate() { - let deserialized: BuildEvent = serde_json::from_str(line).unwrap(); - assert_eq!(deserialized, events[i]); - } + let parsed0: serde_json::Value = serde_json::from_str(&lines[0]).unwrap(); + assert_eq!(parsed0["event"]["type"], "build_started"); + + let parsed1: serde_json::Value = serde_json::from_str(&lines[1]).unwrap(); + assert_eq!(parsed1["event"]["type"], "target_started"); } #[tokio::test] @@ -185,12 +298,15 @@ mod tests { // Create writer (should truncate) let writer = JsonFileWriter::new(&file_path).unwrap(); - let event = BuildEvent::BuildStarted(BuildStarted { + let event = BuildEvent { invocation_id: "new".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - }); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "new".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }; writer.on_event(&event).await.unwrap(); writer.on_close().await.unwrap(); diff --git a/crates/build-events/src/subscribers/logger.rs b/crates/build-events/src/subscribers/logger.rs index 3313494ba8..82c8e5c7c9 100644 --- a/crates/build-events/src/subscribers/logger.rs +++ b/crates/build-events/src/subscribers/logger.rs @@ -2,7 +2,7 @@ //! //! Logs build events using the tracing crate with structured fields. -use crate::events::BuildEvent; +use crate::events::{build_event, BuildEvent}; use crate::subscriber::{BuildEventSubscriber, SubscriberError}; use async_trait::async_trait; use tracing::info; @@ -44,8 +44,8 @@ impl Default for LoggerSubscriber { #[async_trait] impl BuildEventSubscriber for LoggerSubscriber { async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { - match event { - BuildEvent::BuildStarted(e) => { + match &event.event { + Some(build_event::Event::BuildStarted(e)) => { info!( event = "build_started", invocation_id = %e.invocation_id, @@ -55,7 +55,7 @@ impl BuildEventSubscriber for LoggerSubscriber { "Build started" ); } - BuildEvent::BuildFinished(e) => { + Some(build_event::Event::BuildFinished(e)) => { info!( event = "build_finished", invocation_id = %e.invocation_id, @@ -65,7 +65,7 @@ impl BuildEventSubscriber for LoggerSubscriber { "Build finished" ); } - BuildEvent::TargetStarted(e) => { + Some(build_event::Event::TargetStarted(e)) => { info!( event = "target_started", label = %e.label, @@ -74,7 +74,7 @@ impl BuildEventSubscriber for LoggerSubscriber { "Target started" ); } - BuildEvent::TargetCompleted(e) => { + Some(build_event::Event::TargetCompleted(e)) => { info!( event = "target_completed", label = %e.label, @@ -85,7 +85,7 @@ impl BuildEventSubscriber for LoggerSubscriber { "Target completed" ); } - BuildEvent::ActionStarted(e) => { + Some(build_event::Event::ActionStarted(e)) => { info!( event = "action_started", action_id = %e.action_id, @@ -95,7 +95,7 @@ impl BuildEventSubscriber for LoggerSubscriber { "Action started" ); } - BuildEvent::ActionCompleted(e) => { + Some(build_event::Event::ActionCompleted(e)) => { info!( event = "action_completed", action_id = %e.action_id, @@ -107,13 +107,16 @@ impl BuildEventSubscriber for LoggerSubscriber { "Action completed" ); } - BuildEvent::BuildMetadata(e) => { + Some(build_event::Event::BuildMetadata(e)) => { info!( event = "build_metadata", metadata = ?e.metadata, "Build metadata" ); } + None => { + info!("Received event with no inner event type"); + } } Ok(()) @@ -134,12 +137,15 @@ mod tests { let subscriber = LoggerSubscriber::new(); assert_eq!(subscriber.name(), "LoggerSubscriber"); - let event = BuildEvent::BuildStarted(BuildStarted { + let event = BuildEvent { invocation_id: "test-123".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - }); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }; let result = subscriber.on_event(&event).await; assert!(result.is_ok()); @@ -150,18 +156,24 @@ mod tests { let subscriber = LoggerSubscriber::new(); let events = vec![ - BuildEvent::BuildStarted(BuildStarted { + BuildEvent { invocation_id: "test".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - }), - BuildEvent::TargetStarted(TargetStarted { - target_id: "target-1".to_string(), - label: "//foo:bar".to_string(), - target_kind: TargetKind::Binary, - timestamp_millis: current_millis(), - }), + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }, + BuildEvent { + invocation_id: "test".to_string(), + event: Some(build_event::Event::TargetStarted(TargetStarted { + target_id: "target-1".to_string(), + label: "//foo:bar".to_string(), + target_kind: TargetKind::Binary as i32, + timestamp_millis: current_millis(), + })), + }, ]; for event in events { diff --git a/crates/build-events/tests/integration_test.rs b/crates/build-events/tests/integration_test.rs index ef2fa202f2..0000efd9ae 100644 --- a/crates/build-events/tests/integration_test.rs +++ b/crates/build-events/tests/integration_test.rs @@ -6,8 +6,8 @@ use async_trait::async_trait; use build_events::{ BuildEvent, BuildEventBus, BuildEventDispatcher, BuildEventSubscriber, SubscriberError, events::{ - ActionCompleted, ActionStarted, BuildFinished, BuildStarted, TargetCompleted, TargetKind, - TargetStarted, current_millis, + build_event, ActionCompleted, ActionStarted, BuildFinished, BuildStarted, TargetCompleted, + TargetKind, TargetStarted, current_millis, }, subscribers::{JsonFileWriter, LoggerSubscriber}, }; @@ -47,52 +47,70 @@ async fn test_full_event_flow() { }); // Emit all event types - bus.emit(BuildEvent::BuildStarted(BuildStarted { + bus.emit(BuildEvent { invocation_id: "test-integration".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - })); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test-integration".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }); - bus.emit(BuildEvent::TargetStarted(TargetStarted { - target_id: "target-1".to_string(), - label: "//foo:bar".to_string(), - target_kind: TargetKind::Binary, - timestamp_millis: current_millis(), - })); + bus.emit(BuildEvent { + invocation_id: "test-integration".to_string(), + event: Some(build_event::Event::TargetStarted(TargetStarted { + target_id: "target-1".to_string(), + label: "//foo:bar".to_string(), + target_kind: TargetKind::Binary as i32, + timestamp_millis: current_millis(), + })), + }); - bus.emit(BuildEvent::ActionStarted(ActionStarted { - action_id: "action-1".to_string(), - action_name: "CppCompile".to_string(), - target_id: "target-1".to_string(), - timestamp_millis: current_millis(), - })); + bus.emit(BuildEvent { + invocation_id: "test-integration".to_string(), + event: Some(build_event::Event::ActionStarted(ActionStarted { + action_id: "action-1".to_string(), + action_name: "CppCompile".to_string(), + target_id: "target-1".to_string(), + timestamp_millis: current_millis(), + })), + }); - bus.emit(BuildEvent::ActionCompleted(ActionCompleted { - action_id: "action-1".to_string(), - success: true, - timestamp_millis: current_millis(), - exit_code: 0, - stdout: Some("compiled successfully".to_string()), - stderr: None, - })); + bus.emit(BuildEvent { + invocation_id: "test-integration".to_string(), + event: Some(build_event::Event::ActionCompleted(ActionCompleted { + action_id: "action-1".to_string(), + success: true, + timestamp_millis: current_millis(), + exit_code: 0, + stdout: Some("compiled successfully".to_string()), + stderr: None, + })), + }); - bus.emit(BuildEvent::TargetCompleted(TargetCompleted { - target_id: "target-1".to_string(), - label: "//foo:bar".to_string(), - success: true, - timestamp_millis: current_millis(), - error_message: None, - outputs: vec!["foo/bar".to_string()], - cache_hit: false, - })); + bus.emit(BuildEvent { + invocation_id: "test-integration".to_string(), + event: Some(build_event::Event::TargetCompleted(TargetCompleted { + target_id: "target-1".to_string(), + label: "//foo:bar".to_string(), + success: true, + timestamp_millis: current_millis(), + error_message: None, + outputs: vec!["foo/bar".to_string()], + cache_hit: false, + })), + }); - bus.emit(BuildEvent::BuildFinished(BuildFinished { + bus.emit(BuildEvent { invocation_id: "test-integration".to_string(), - success: true, - timestamp_millis: current_millis(), - error_message: None, - })); + event: Some(build_event::Event::BuildFinished(BuildFinished { + invocation_id: "test-integration".to_string(), + success: true, + timestamp_millis: current_millis(), + error_message: None, + })), + }); // Wait for events to process tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; @@ -126,12 +144,15 @@ async fn test_multiple_subscribers() { // Emit a few events for i in 0..3 { - bus.emit(BuildEvent::BuildStarted(BuildStarted { + bus.emit(BuildEvent { invocation_id: format!("test-{}", i), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - })); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: format!("test-{}", i), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }); } tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; @@ -146,9 +167,9 @@ async fn test_multiple_subscribers() { let lines: Vec<&str> = content.lines().collect(); assert_eq!(lines.len(), 3); - // Verify each line is valid JSON + // Verify each line is valid JSON (we can parse as serde_json::Value even though BuildEvent doesn't have serde) for line in lines { - let _: BuildEvent = serde_json::from_str(line).unwrap(); + let _: serde_json::Value = serde_json::from_str(line).unwrap(); } } @@ -157,12 +178,15 @@ async fn test_late_subscriber_misses_early_events() { let bus = BuildEventBus::new(100); // Emit event before subscription - bus.emit(BuildEvent::BuildStarted(BuildStarted { + bus.emit(BuildEvent { invocation_id: "early".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - })); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "early".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }); // Subscribe after emission let count = Arc::new(AtomicUsize::new(0)); @@ -176,12 +200,15 @@ async fn test_late_subscriber_misses_early_events() { }); // Emit event after subscription - bus.emit(BuildEvent::BuildStarted(BuildStarted { + bus.emit(BuildEvent { invocation_id: "late".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - })); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "late".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }); tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; drop(bus); @@ -207,20 +234,26 @@ async fn test_bus_cloning() { }); // Emit from original bus - bus.emit(BuildEvent::BuildStarted(BuildStarted { + bus.emit(BuildEvent { invocation_id: "original".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - })); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "original".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }); // Emit from cloned bus - bus_clone.emit(BuildEvent::BuildStarted(BuildStarted { + bus_clone.emit(BuildEvent { invocation_id: "clone".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - })); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "clone".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }); tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; drop(bus); diff --git a/crates/build-sandbox/Cargo.toml b/crates/build-sandbox/Cargo.toml index 48171a5d81..f0c82771cb 100644 --- a/crates/build-sandbox/Cargo.toml +++ b/crates/build-sandbox/Cargo.toml @@ -15,8 +15,8 @@ tracing.workspace = true common.workspace = true +# Use build-events which now provides proto types +build-events.workspace = true + [target.'cfg(target_os = "linux")'.dependencies] hakoniwa.workspace = true - -# TODO, get this out of the build-sandbox crate -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251010205210-42ae8fd1517e.1", registry = "buf" } diff --git a/crates/build-sandbox/src/executor.rs b/crates/build-sandbox/src/executor.rs index 0c4871ee79..a8a4993e42 100644 --- a/crates/build-sandbox/src/executor.rs +++ b/crates/build-sandbox/src/executor.rs @@ -1,7 +1,7 @@ -use hakoniwa::Container; -use minimal_spongebob_community_neoeinstein_prost::spongebob::v1::{ - BuildEvent, TargetCompleted, TargetKind, TargetStarted, build_event, +use build_events::events::{ + build_event, BuildEvent, TargetCompleted, TargetKind, TargetStarted, }; +use hakoniwa::Container; use std::fs; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index f898d61a10..f38daa257f 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -2,7 +2,7 @@ use crate::{Error, lockfile::PrebuiltsLock, remote_storage::RemoteStorage, run:: use crate::{GlobalArgs, PackagesArg}; use anyhow::Context; use build_events::events::{ - BuildEvent, BuildFinished, BuildMetadata, BuildStarted, current_millis, + build_event, BuildEvent, BuildFinished, BuildMetadata, BuildStarted, current_millis, }; use build_events::{BuildEventBus, BuildEventDispatcher}; use build_events_proto::SpongeBobSubscriberV2; @@ -95,61 +95,57 @@ pub async fn cmd_build_impl( .to_string(); // Emit BuildStarted event - event_bus.emit(BuildEvent::BuildStarted(BuildStarted { + event_bus.emit(BuildEvent { invocation_id: invocation_id.clone(), - command, - timestamp_millis: current_millis(), - working_directory, - })); + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: invocation_id.clone(), + command, + timestamp_millis: current_millis(), + working_directory, + })), + }); // Collect and emit git metadata let mut metadata = HashMap::new(); // Get git user - if let Ok(output) = Command::new("git").args(["config", "user.name"]).output() { - if output.status.success() { - if let Ok(user) = String::from_utf8(output.stdout) { + if let Ok(output) = Command::new("git").args(["config", "user.name"]).output() + && output.status.success() + && let Ok(user) = String::from_utf8(output.stdout) { metadata.insert("user".to_string(), user.trim().to_string()); } - } - } // Get git branch if let Ok(output) = Command::new("git") .args(["rev-parse", "--abbrev-ref", "HEAD"]) .output() - { - if output.status.success() { - if let Ok(branch) = String::from_utf8(output.stdout) { + && output.status.success() + && let Ok(branch) = String::from_utf8(output.stdout) { metadata.insert("branch".to_string(), branch.trim().to_string()); } - } - } // Get git commit SHA - if let Ok(output) = Command::new("git").args(["rev-parse", "HEAD"]).output() { - if output.status.success() { - if let Ok(commit) = String::from_utf8(output.stdout) { + if let Ok(output) = Command::new("git").args(["rev-parse", "HEAD"]).output() + && output.status.success() + && let Ok(commit) = String::from_utf8(output.stdout) { metadata.insert("commit".to_string(), commit.trim().to_string()); } - } - } // Get git remote URL if let Ok(output) = Command::new("git") .args(["config", "remote.origin.url"]) .output() - { - if output.status.success() { - if let Ok(repo_url) = String::from_utf8(output.stdout) { + && output.status.success() + && let Ok(repo_url) = String::from_utf8(output.stdout) { metadata.insert("repo_url".to_string(), repo_url.trim().to_string()); } - } - } // Emit BuildMetadata event if we collected any metadata if !metadata.is_empty() { - event_bus.emit(BuildEvent::BuildMetadata(BuildMetadata { metadata })); + event_bus.emit(BuildEvent { + invocation_id: invocation_id.clone(), + event: Some(build_event::Event::BuildMetadata(BuildMetadata { metadata })), + }); } let build_success = match (globals.no_cache, globals.no_fetch) { @@ -198,12 +194,15 @@ pub async fn cmd_build_impl( let error_message = build_success.as_ref().err().map(|e| e.to_string()); // Emit BuildFinished event - event_bus.emit(BuildEvent::BuildFinished(BuildFinished { + event_bus.emit(BuildEvent { invocation_id: invocation_id.clone(), - success: build_succeeded, - timestamp_millis: current_millis(), - error_message, - })); + event: Some(build_event::Event::BuildFinished(BuildFinished { + invocation_id: invocation_id.clone(), + success: build_succeeded, + timestamp_millis: current_millis(), + error_message, + })), + }); // Propagate error if build failed build_success.context("Failed to execute build")?; From e33c708a3120054b1db18654913b0988fc564b28 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Tue, 14 Oct 2025 13:42:48 -0700 Subject: [PATCH 15/39] refactor --- Cargo.lock | 1 - .../src/spongebob_subscriber_v2.rs | 72 ++++++++- crates/build-sandbox/Cargo.toml | 1 - crates/build-sandbox/src/executor.rs | 142 ++++++++---------- crates/build-sandbox/src/lib.rs | 11 +- crates/minimal/src/cmd_build.rs | 16 +- crates/minimal/src/cmd_patched_build.rs | 21 ++- crates/minimal/src/run.rs | 55 ++----- 8 files changed, 179 insertions(+), 140 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db9e63a802..6976163d61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -384,7 +384,6 @@ dependencies = [ "hakoniwa", "nickel-lang-core", "serde", - "spongebob", "thiserror 2.0.17", "tokio", "tracing", diff --git a/crates/build-events-proto/src/spongebob_subscriber_v2.rs b/crates/build-events-proto/src/spongebob_subscriber_v2.rs index 88d0ca4863..4771a156a3 100644 --- a/crates/build-events-proto/src/spongebob_subscriber_v2.rs +++ b/crates/build-events-proto/src/spongebob_subscriber_v2.rs @@ -34,6 +34,8 @@ use tracing::warn; #[cfg(feature = "spongebob-subscriber")] pub struct SpongeBobSubscriberV2 { invocation: std::sync::Arc>, + /// Cache of file paths from BuildMetadata events, keyed by "{target_id}/stdout_path" etc. + file_paths: std::sync::Arc>>, } #[cfg(feature = "spongebob-subscriber")] @@ -57,6 +59,9 @@ impl SpongeBobSubscriberV2 { pub fn from_invocation(invocation: spongebob::SpongeBobInvocation) -> Self { Self { invocation: std::sync::Arc::new(tokio::sync::Mutex::new(invocation)), + file_paths: std::sync::Arc::new(tokio::sync::Mutex::new( + std::collections::HashMap::new(), + )), } } } @@ -65,19 +70,76 @@ impl SpongeBobSubscriberV2 { #[async_trait] impl BuildEventSubscriber for SpongeBobSubscriberV2 { async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { - // Get lock on invocation - let mut invocation = self.invocation.lock().await; + use build_events::events::build_event::Event; + + // Handle BuildMetadata events to cache file paths + if let Some(Event::BuildMetadata(metadata)) = &event.event { + let mut file_paths = self.file_paths.lock().await; + for (key, value) in &metadata.metadata { + if key.ends_with("_path") { + file_paths.insert(key.clone(), value.clone()); + } + } + // Don't publish BuildMetadata with file paths to avoid clutter + return Ok(()); + } + + // Handle TargetCompleted events with file uploads + if let Some(Event::TargetCompleted(target_completed)) = &event.event { + let target_id = &target_completed.target_id; - // Publish event + // Get file paths from cache + let file_paths = self.file_paths.lock().await; + let stdout_key = format!("{}/stdout_path", target_id); + let stderr_key = format!("{}/stderr_path", target_id); + + let stdout_path = file_paths.get(&stdout_key); + let stderr_path = file_paths.get(&stderr_key); + + // First publish the TargetCompleted event + let mut invocation = self.invocation.lock().await; + if let Err(e) = invocation.publish_build_event(event.clone()).await { + warn!("Failed to publish TargetCompleted event to Spongebob: {}", e); + return Err(SubscriberError::Custom(format!( + "Failed to publish TargetCompleted event: {}", + e + ))); + } + + // Then upload files if paths are available + if let Some(stdout_path) = stdout_path { + if let Ok(contents) = std::fs::read(stdout_path) { + if let Err(e) = invocation.upload_file(target_id, "stdout", contents).await { + warn!("Failed to upload stdout for target {}: {}", target_id, e); + } + } else { + warn!("Failed to read stdout file: {}", stdout_path); + } + } + + if let Some(stderr_path) = stderr_path { + if let Ok(contents) = std::fs::read(stderr_path) { + if let Err(e) = invocation.upload_file(target_id, "stderr", contents).await { + warn!("Failed to upload stderr for target {}: {}", target_id, e); + } + } else { + warn!("Failed to read stderr file: {}", stderr_path); + } + } + + drop(invocation); + return Ok(()); + } + + // For all other events, just publish them + let mut invocation = self.invocation.lock().await; match invocation.publish_build_event(event.clone()).await { Ok(()) => { tracing::debug!("Successfully published build event to Spongebob"); Ok(()) } Err(e) => { - // Log warning but don't fail the build warn!("Failed to publish build event to Spongebob: {}", e); - // Return error so dispatcher can decide how to handle it Err(SubscriberError::Custom(format!( "Failed to publish event: {}", e diff --git a/crates/build-sandbox/Cargo.toml b/crates/build-sandbox/Cargo.toml index f0c82771cb..e604ce6834 100644 --- a/crates/build-sandbox/Cargo.toml +++ b/crates/build-sandbox/Cargo.toml @@ -8,7 +8,6 @@ publish.workspace = true globset.workspace = true nickel-lang-core.workspace = true serde.workspace = true -spongebob = { path = "../spongebob" } thiserror.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/crates/build-sandbox/src/executor.rs b/crates/build-sandbox/src/executor.rs index a8a4993e42..a0631f3fab 100644 --- a/crates/build-sandbox/src/executor.rs +++ b/crates/build-sandbox/src/executor.rs @@ -1,7 +1,9 @@ use build_events::events::{ - build_event, BuildEvent, TargetCompleted, TargetKind, TargetStarted, + build_event, BuildEvent, BuildMetadata, TargetCompleted, TargetKind, TargetStarted, }; +use build_events::BuildEventBus; use hakoniwa::Container; +use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -55,13 +57,14 @@ impl BuildExecutor { Ok(executor) } - #[tracing::instrument(skip(config, spongebob_invocation), fields(indicatif.pb_hide, package = %config.name))] + #[tracing::instrument(skip(config, event_bus), fields(indicatif.pb_hide, package = %config.name))] pub async fn execute( &self, config: &BuildConfig, - spongebob_invocation: &mut Option, + event_bus: &BuildEventBus, + invocation_id: &str, target_id: &str, - ) -> Result<(i32, Option)> { + ) -> Result { info!( "Linking {} inputs to build environment", config.inputs.len() @@ -83,57 +86,65 @@ impl BuildExecutor { } // Emit TargetStarted event - if let Some(invocation) = spongebob_invocation { - let target_started = BuildEvent { - invocation_id: invocation.invocation_id().to_string(), - event: Some(build_event::Event::TargetStarted(TargetStarted { - target_id: target_id.to_string(), - label: config.name.clone(), - target_kind: TargetKind::Binary as i32, - timestamp_millis: current_millis(), - })), - }; - if let Err(e) = invocation.publish_build_event(target_started).await { - warn!("Failed to publish TargetStarted event: {}", e); - } - } - - let execute_result = self - .execute_in_container(config, spongebob_invocation, target_id) - .await; + event_bus.emit(BuildEvent { + invocation_id: invocation_id.to_string(), + event: Some(build_event::Event::TargetStarted(TargetStarted { + target_id: target_id.to_string(), + label: config.name.clone(), + target_kind: TargetKind::Binary as i32, + timestamp_millis: current_millis(), + })), + }); + + let execute_result = self.execute_in_container(config, target_id).await; // Emit TargetCompleted event - if let Some(invocation) = spongebob_invocation { - let (success, error_message) = match &execute_result { - Ok(_) => (true, None), - Err(e) => (false, Some(e.to_string())), - }; + let (success, error_message) = match &execute_result { + Ok(_) => (true, None), + Err(e) => (false, Some(e.to_string())), + }; - let target_completed = BuildEvent { - invocation_id: invocation.invocation_id().to_string(), - event: Some(build_event::Event::TargetCompleted(TargetCompleted { - target_id: target_id.to_string(), - label: config.name.clone(), - success, - timestamp_millis: current_millis(), - error_message, - outputs: vec![], // Outputs are collected later - cache_hit: false, // Cache hit detection happens earlier - })), - }; - if let Err(e) = invocation.publish_build_event(target_completed).await { - warn!("Failed to publish TargetCompleted event: {}", e); - } - } + event_bus.emit(BuildEvent { + invocation_id: invocation_id.to_string(), + event: Some(build_event::Event::TargetCompleted(TargetCompleted { + target_id: target_id.to_string(), + label: config.name.clone(), + success, + timestamp_millis: current_millis(), + error_message, + outputs: vec![], // Outputs are collected later + cache_hit: false, // Cache hit detection happens earlier + })), + }); + + // Emit BuildMetadata with file paths so subscriber can upload them + let mut file_metadata = HashMap::new(); + file_metadata.insert( + format!("{}/stdout_path", target_id), + self.build_workspace_dir + .join("stdout") + .to_string_lossy() + .to_string(), + ); + file_metadata.insert( + format!("{}/stderr_path", target_id), + self.build_workspace_dir + .join("stderr") + .to_string_lossy() + .to_string(), + ); + + event_bus.emit(BuildEvent { + invocation_id: invocation_id.to_string(), + event: Some(build_event::Event::BuildMetadata(BuildMetadata { + metadata: file_metadata, + })), + }); // Return the original result execute_result?; - let spongebob_url = spongebob_invocation - .as_ref() - .map(|inv| inv.url().to_string()); - - Ok((0, spongebob_url)) + Ok(0) } pub fn build_workspace_dir(&self) -> &Path { @@ -187,13 +198,8 @@ impl BuildExecutor { Ok(()) } - #[tracing::instrument(skip(config, spongebob_invocation), fields(indicatif.pb_show))] - async fn execute_in_container( - &self, - config: &BuildConfig, - spongebob_invocation: &mut Option, - target_id: &str, - ) -> Result<()> { + #[tracing::instrument(skip(config), fields(indicatif.pb_show))] + async fn execute_in_container(&self, config: &BuildConfig, target_id: &str) -> Result<()> { let rootfs = self.prepare_rootfs(config)?; let sandbox_mount_point = "/build"; let program = &config.build_script.executable.to_string_lossy(); @@ -265,28 +271,6 @@ impl BuildExecutor { } })?; - if let Some(invocation) = spongebob_invocation { - if let Err(e) = invocation - .upload_file(target_id, "stdout", output.stdout.to_vec()) - .await - { - warn!( - "Failed to upload stdout to SpongeBob for {}: {}", - config.name, e - ); - } - - if let Err(e) = invocation - .upload_file(target_id, "stderr", output.stderr.to_vec()) - .await - { - warn!( - "Failed to upload stderr to SpongeBob for {}: {}", - config.name, e - ); - } - } - if !output.status.success() { let exit_code = output.status.code; @@ -312,9 +296,7 @@ impl BuildExecutor { return Err(ExecutionError::BuildFailed { code: exit_code, temp_dir: self.build_workspace_dir.clone(), - spongebob_url: spongebob_invocation - .as_ref() - .map(|inv| inv.url().to_string()), + spongebob_url: None, // URL is now tracked at invocation level } .into()); } diff --git a/crates/build-sandbox/src/lib.rs b/crates/build-sandbox/src/lib.rs index 8556460b56..1b1901170d 100644 --- a/crates/build-sandbox/src/lib.rs +++ b/crates/build-sandbox/src/lib.rs @@ -15,7 +15,6 @@ pub use output::OutputValidator; pub struct BuildResult { pub exit_code: i32, pub outputs: Vec, - pub spongebob_url: Option, } /// Run a build in a sandboxed environment @@ -23,19 +22,22 @@ pub struct BuildResult { /// # Parameters /// * `config` - Build configuration including inputs, outputs, and build script /// * `cache_dest_dir` - Final destination in cache where successful build outputs are stored +/// * `event_bus` - Event bus for emitting build events +/// * `invocation_id` - Invocation ID for this build session /// * `sandbox_base_dir` - Base directory for creating temporary build sandboxes /// * `target_id` - Unique identifier for the target being built #[tracing::instrument(skip_all, fields(name = config.name, indicatif.pb_show))] pub async fn run_build( config: &BuildConfig, cache_dest_dir: &std::path::Path, - spongebob_invocation: &mut Option, + event_bus: &build_events::BuildEventBus, + invocation_id: &str, sandbox_base_dir: std::path::PathBuf, target_id: &str, ) -> Result { let executor = BuildExecutor::new(sandbox_base_dir, config.name.clone())?; - let (exit_code, spongebob_url) = executor - .execute(config, spongebob_invocation, target_id) + let exit_code = executor + .execute(config, event_bus, invocation_id, target_id) .await?; let outputs = OutputValidator::validate_and_collect( @@ -47,6 +49,5 @@ pub async fn run_build( Ok(BuildResult { exit_code, outputs, - spongebob_url, }) } diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index f38daa257f..d6709dd6f9 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -40,7 +40,7 @@ pub async fn cmd_build_impl( .await .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; - let mut spongebob_invocation = Some(spongebob_client.create_invocation()); + let spongebob_invocation = Some(spongebob_client.create_invocation()); rayon::ThreadPoolBuilder::new() .num_threads(num_parallel_builds) @@ -151,7 +151,7 @@ pub async fn cmd_build_impl( let build_success = match (globals.no_cache, globals.no_fetch) { // No local or remote cache (true, true) => { - run.execute(ExecPlan::new(graph), None, &mut spongebob_invocation) + run.execute(ExecPlan::new(graph), None, &event_bus, &invocation_id) .await } // Both caches @@ -162,7 +162,8 @@ pub async fn cmd_build_impl( run.execute( ExecPlan::new_with_bin_provider(graph, (local_adapter, remote_adapter)), Some(&remote_cache), - &mut spongebob_invocation, + &event_bus, + &invocation_id, ) .await } @@ -173,7 +174,8 @@ pub async fn cmd_build_impl( run.execute( ExecPlan::new_with_bin_provider(graph, remote_adapter), Some(&remote_cache), - &mut spongebob_invocation, + &event_bus, + &invocation_id, ) .await } @@ -183,7 +185,8 @@ pub async fn cmd_build_impl( run.execute( ExecPlan::new_with_bin_provider(graph, local_adapter), None, - &mut spongebob_invocation, + &event_bus, + &invocation_id, ) .await } @@ -257,11 +260,10 @@ pub async fn cmd_build_impl( } } + // Display build summary with spongebob URL let spongebob_url = spongebob_invocation .as_ref() .map(|inv| inv.url().to_string()); - - // Display build summary display_build_summary(graph, &cache, globals, &run, spongebob_url.as_deref()); Ok(()) diff --git a/crates/minimal/src/cmd_patched_build.rs b/crates/minimal/src/cmd_patched_build.rs index 2864e765de..940b97f657 100644 --- a/crates/minimal/src/cmd_patched_build.rs +++ b/crates/minimal/src/cmd_patched_build.rs @@ -105,14 +105,31 @@ pub async fn cmd_patched_build(args: PatchedBuildArgs, globals: &GlobalArgs) -> .await .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; + let spongebob_invocation = spongebob_client.create_invocation(); + let invocation_id = spongebob_invocation.invocation_id().to_string(); + + // Setup event bus and subscriber + use build_events::{BuildEventBus, BuildEventDispatcher}; + use build_events_proto::SpongeBobSubscriberV2; + + let event_bus = BuildEventBus::new(10000); + let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); + let subscriber = SpongeBobSubscriberV2::from_invocation(spongebob_invocation.clone()); + dispatcher.add_subscriber(Box::new(subscriber)); + + // Spawn dispatcher in background + tokio::spawn(async move { + dispatcher.run().await; + }); + // Use package name as target ID for semantic meaning let target_id = build.name.clone(); - let mut spongebob_invocation = Some(spongebob_client.create_invocation()); run_build( &config, out_dir.path(), - &mut spongebob_invocation, + &event_bus, + &invocation_id, output_base.clone(), &target_id, ) diff --git a/crates/minimal/src/run.rs b/crates/minimal/src/run.rs index c51aab09fe..be0555adfd 100644 --- a/crates/minimal/src/run.rs +++ b/crates/minimal/src/run.rs @@ -359,7 +359,6 @@ pub struct Run<'a> { remote_storage: RemoteStorage, lockfile: PrebuiltsLock, output_base: PathBuf, - latest_spongebob_url: Option, } impl<'a> Run<'a> { @@ -376,7 +375,6 @@ impl<'a> Run<'a> { remote_storage, lockfile, output_base, - latest_spongebob_url: None, } } @@ -457,16 +455,16 @@ impl<'a> Run<'a> { /// runs a single isolated build, does not take self so it can be spawned in a thread. /// - /// Upon success, returns both the pending cache entry representing the build outputs, - /// and the spongebob URL of the build. + /// Upon success, returns the pending cache entry representing the build outputs. #[tracing::instrument(skip_all, fields(indicatif.pb_hide))] async fn do_build( &self, build: &BuildSpecRef, _full_build: bool, remote_cache: Option<&RemoteCache>, - spongebob_invocation: &mut Option, - ) -> Result<(Option, Option)> { + event_bus: &build_events::BuildEventBus, + invocation_id: &str, + ) -> Result> { let bsh = self.graph.spec_hash(build); let build = self.graph.get(build).unwrap(); @@ -478,7 +476,7 @@ impl<'a> Run<'a> { e.path().display(), ); // Return None to indicate this was a cache hit, not a new build - return Ok((None, None)); + return Ok(None); } if build.is_pure_prebuilt() { info!("Materializing prebuilt package: {}", build.name); @@ -491,7 +489,7 @@ impl<'a> Run<'a> { ) .await?; info!("Successfully materialized prebuilt package: {}", build.name); - return Ok((Some(result), None)); + return Ok(Some(result)); } let (dependencies, inputs, temp_dirs) = self @@ -504,7 +502,7 @@ impl<'a> Run<'a> { build.name ); let cache_handle = self.cache.write_dir(&bsh).unwrap(); - Ok((Some(cache_handle), None)) + Ok(Some(cache_handle)) } else { // Regular build with build script let cmd_parts: Vec = @@ -543,24 +541,22 @@ impl<'a> Run<'a> { let target_id = build.name.clone(); info!("Building package: {}", build.name); - let build_result = run_build( + let _build_result = run_build( &config, out_dir.path(), - spongebob_invocation, + event_bus, + invocation_id, self.output_base.clone(), &target_id, ) .await .map_err(|e| anyhow::anyhow!("Failed to build {}: {}", build.name, e))?; - // Extract SpongeBob URL from BuildResult - let spongebob_url = build_result.spongebob_url; - info!("Successfully built package: {}", build.name); for tempdir in temp_dirs.into_iter() { drop(tempdir); } - Ok((Some(out_dir), spongebob_url)) + Ok(Some(out_dir)) } } @@ -568,7 +564,8 @@ impl<'a> Run<'a> { &mut self, plan: ExecPlan<'a, BP>, remote_cache: Option<&RemoteCache>, - spongebob_invocation: &mut Option, + event_bus: &build_events::BuildEventBus, + invocation_id: &str, ) -> Result<()> { // Execute builds in dependency order - each build runs in isolation // and can only access outputs from previously completed builds @@ -582,7 +579,6 @@ impl<'a> Run<'a> { let cache_handles = Arc::new(Mutex::new(Vec::with_capacity( phase.as_ref().unwrap().builds.len(), ))); - let spongebob_urls = Arc::new(Mutex::new(Vec::::new())); rayon::scope(|s| { for (bsr, full_build) in phase.unwrap().builds.iter() { @@ -592,8 +588,6 @@ impl<'a> Run<'a> { let self2 = self2.clone(); let err_bsr = build_which_errored.clone(); let cache_handles = cache_handles.clone(); - let spongebob_urls = spongebob_urls.clone(); - let mut invocation_clone = spongebob_invocation.clone(); s.spawn(move |_| { let _rt = tokio_runtime.enter(); @@ -601,23 +595,21 @@ impl<'a> Run<'a> { &bsr, full_build, remote_cache, - &mut invocation_clone, + event_bus, + invocation_id, )); match result { Err(e) => { *err_bsr.lock().unwrap() = Some((bsr, e)); } - Ok((cache_handle, spongebob_url)) => { + Ok(cache_handle) => { if let Some(cache_handle) = cache_handle { cache_handles.lock().unwrap().push(( cache_handle, self2.graph.get(&bsr).unwrap().name.clone(), )); } - if let Some(url) = spongebob_url { - spongebob_urls.lock().unwrap().push(url); - } } } }); @@ -645,26 +637,11 @@ impl<'a> Run<'a> { .unwrap() }); - // Store the latest SpongeBob URL (if any) for display at the end - let urls = Arc::into_inner(spongebob_urls) - .unwrap() - .into_inner() - .unwrap(); - if let Some(latest_url) = urls.last() { - self.latest_spongebob_url = Some(latest_url.clone()); - } - err.map(|(_bsr, e)| Err(e)).unwrap_or(Ok(()))?; } Ok(()) } - - /// Get the latest SpongeBob URL from the most recent build session - #[allow(dead_code)] - pub fn latest_spongebob_url(&self) -> Option<&String> { - self.latest_spongebob_url.as_ref() - } } fn extract_prebuilt_archive(archive_path: &PathBuf, extract_dir: &Path) -> Result<()> { From 77a395ac9ea16c4736d1773cd0977b41f6044562 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Tue, 14 Oct 2025 13:49:44 -0700 Subject: [PATCH 16/39] refactor --- crates/build-events/src/lib.rs | 37 ++++++++++++ crates/build-sandbox/src/executor.rs | 27 ++++----- crates/build-sandbox/src/lib.rs | 8 +-- crates/minimal/src/cmd_build.rs | 79 ++++++------------------- crates/minimal/src/cmd_patched_build.rs | 33 +---------- crates/minimal/src/main.rs | 23 +++++++ crates/minimal/src/run.rs | 15 +---- 7 files changed, 95 insertions(+), 127 deletions(-) diff --git a/crates/build-events/src/lib.rs b/crates/build-events/src/lib.rs index 685884958d..16896f8bd9 100644 --- a/crates/build-events/src/lib.rs +++ b/crates/build-events/src/lib.rs @@ -131,3 +131,40 @@ pub use bus::BuildEventBus; pub use dispatcher::BuildEventDispatcher; pub use events::BuildEvent; pub use subscriber::{BuildEventSubscriber, SubscriberError}; + +// Global event bus infrastructure +use std::sync::OnceLock; + +static GLOBAL_EVENT_BUS: OnceLock = OnceLock::new(); +static INVOCATION_ID: OnceLock = OnceLock::new(); + +/// Initialize the global event bus and invocation ID +/// +/// This should be called once at application startup before any events are emitted. +/// Panics if called more than once. +pub fn initialize_global_event_bus(invocation_id: String) { + GLOBAL_EVENT_BUS + .set(BuildEventBus::new(10000)) + .expect("Global event bus already initialized"); + INVOCATION_ID + .set(invocation_id) + .expect("Invocation ID already initialized"); +} + +/// Get a reference to the global event bus +/// +/// Panics if the global event bus has not been initialized via `initialize_global_event_bus`. +pub fn event_bus() -> &'static BuildEventBus { + GLOBAL_EVENT_BUS + .get() + .expect("Global event bus not initialized. Call initialize_global_event_bus() first.") +} + +/// Get the invocation ID for this build session +/// +/// Panics if the invocation ID has not been initialized via `initialize_global_event_bus`. +pub fn invocation_id() -> &'static str { + INVOCATION_ID + .get() + .expect("Invocation ID not initialized. Call initialize_global_event_bus() first.") +} diff --git a/crates/build-sandbox/src/executor.rs b/crates/build-sandbox/src/executor.rs index a0631f3fab..f34ac6827e 100644 --- a/crates/build-sandbox/src/executor.rs +++ b/crates/build-sandbox/src/executor.rs @@ -1,7 +1,6 @@ use build_events::events::{ build_event, BuildEvent, BuildMetadata, TargetCompleted, TargetKind, TargetStarted, }; -use build_events::BuildEventBus; use hakoniwa::Container; use std::collections::HashMap; use std::fs; @@ -57,14 +56,8 @@ impl BuildExecutor { Ok(executor) } - #[tracing::instrument(skip(config, event_bus), fields(indicatif.pb_hide, package = %config.name))] - pub async fn execute( - &self, - config: &BuildConfig, - event_bus: &BuildEventBus, - invocation_id: &str, - target_id: &str, - ) -> Result { + #[tracing::instrument(skip(config), fields(indicatif.pb_hide, package = %config.name))] + pub async fn execute(&self, config: &BuildConfig, target_id: &str) -> Result { info!( "Linking {} inputs to build environment", config.inputs.len() @@ -85,9 +78,9 @@ impl BuildExecutor { } } - // Emit TargetStarted event - event_bus.emit(BuildEvent { - invocation_id: invocation_id.to_string(), + // Emit TargetStarted event using global event bus + build_events::event_bus().emit(BuildEvent { + invocation_id: build_events::invocation_id().to_string(), event: Some(build_event::Event::TargetStarted(TargetStarted { target_id: target_id.to_string(), label: config.name.clone(), @@ -98,14 +91,14 @@ impl BuildExecutor { let execute_result = self.execute_in_container(config, target_id).await; - // Emit TargetCompleted event + // Emit TargetCompleted event using global event bus let (success, error_message) = match &execute_result { Ok(_) => (true, None), Err(e) => (false, Some(e.to_string())), }; - event_bus.emit(BuildEvent { - invocation_id: invocation_id.to_string(), + build_events::event_bus().emit(BuildEvent { + invocation_id: build_events::invocation_id().to_string(), event: Some(build_event::Event::TargetCompleted(TargetCompleted { target_id: target_id.to_string(), label: config.name.clone(), @@ -134,8 +127,8 @@ impl BuildExecutor { .to_string(), ); - event_bus.emit(BuildEvent { - invocation_id: invocation_id.to_string(), + build_events::event_bus().emit(BuildEvent { + invocation_id: build_events::invocation_id().to_string(), event: Some(build_event::Event::BuildMetadata(BuildMetadata { metadata: file_metadata, })), diff --git a/crates/build-sandbox/src/lib.rs b/crates/build-sandbox/src/lib.rs index 1b1901170d..e58ffac73b 100644 --- a/crates/build-sandbox/src/lib.rs +++ b/crates/build-sandbox/src/lib.rs @@ -22,23 +22,17 @@ pub struct BuildResult { /// # Parameters /// * `config` - Build configuration including inputs, outputs, and build script /// * `cache_dest_dir` - Final destination in cache where successful build outputs are stored -/// * `event_bus` - Event bus for emitting build events -/// * `invocation_id` - Invocation ID for this build session /// * `sandbox_base_dir` - Base directory for creating temporary build sandboxes /// * `target_id` - Unique identifier for the target being built #[tracing::instrument(skip_all, fields(name = config.name, indicatif.pb_show))] pub async fn run_build( config: &BuildConfig, cache_dest_dir: &std::path::Path, - event_bus: &build_events::BuildEventBus, - invocation_id: &str, sandbox_base_dir: std::path::PathBuf, target_id: &str, ) -> Result { let executor = BuildExecutor::new(sandbox_base_dir, config.name.clone())?; - let exit_code = executor - .execute(config, event_bus, invocation_id, target_id) - .await?; + let exit_code = executor.execute(config, target_id).await?; let outputs = OutputValidator::validate_and_collect( config, diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index d6709dd6f9..309c637a96 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -4,8 +4,6 @@ use anyhow::Context; use build_events::events::{ build_event, BuildEvent, BuildFinished, BuildMetadata, BuildStarted, current_millis, }; -use build_events::{BuildEventBus, BuildEventDispatcher}; -use build_events_proto::SpongeBobSubscriberV2; use cache::{Cache, CacheBinProvider, LocalDir, RemoteBinProvider}; use graph::{DepGraph, ExecPlan, Transitives}; use std::collections::HashMap; @@ -35,13 +33,6 @@ pub async fn cmd_build_impl( cache: Cache, num_parallel_builds: usize, ) -> anyhow::Result<()> { - // Create SpongeBob invocation for this build command - let mut spongebob_client = spongebob::SpongeBob::new() - .await - .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; - - let spongebob_invocation = Some(spongebob_client.create_invocation()); - rayon::ThreadPoolBuilder::new() .num_threads(num_parallel_builds) .build_global() @@ -60,27 +51,6 @@ pub async fn cmd_build_impl( output_base, ); - // Setup build events system - let event_bus = BuildEventBus::new(10000); - - // Create dispatcher with SpongeBobSubscriberV2 if we have an invocation - if let Some(ref invocation) = spongebob_invocation { - let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); - let subscriber = SpongeBobSubscriberV2::from_invocation(invocation.clone()); - dispatcher.add_subscriber(Box::new(subscriber)); - - // Spawn dispatcher in background - tokio::spawn(async move { - dispatcher.run().await; - }); - } - - // Get invocation_id for events - let invocation_id = spongebob_invocation - .as_ref() - .map(|inv| inv.invocation_id().to_string()) - .unwrap_or_else(|| "local".to_string()); - // Get command for BuildStarted event (skip binary path at index 0) let args: Vec = std::env::args().collect(); let command = if args.len() > 1 { @@ -94,11 +64,11 @@ pub async fn cmd_build_impl( .to_string_lossy() .to_string(); - // Emit BuildStarted event - event_bus.emit(BuildEvent { - invocation_id: invocation_id.clone(), + // Emit BuildStarted event using global event bus + build_events::event_bus().emit(BuildEvent { + invocation_id: build_events::invocation_id().to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { - invocation_id: invocation_id.clone(), + invocation_id: build_events::invocation_id().to_string(), command, timestamp_millis: current_millis(), working_directory, @@ -140,20 +110,17 @@ pub async fn cmd_build_impl( metadata.insert("repo_url".to_string(), repo_url.trim().to_string()); } - // Emit BuildMetadata event if we collected any metadata + // Emit BuildMetadata event if we collected any metadata (using global event bus) if !metadata.is_empty() { - event_bus.emit(BuildEvent { - invocation_id: invocation_id.clone(), + build_events::event_bus().emit(BuildEvent { + invocation_id: build_events::invocation_id().to_string(), event: Some(build_event::Event::BuildMetadata(BuildMetadata { metadata })), }); } let build_success = match (globals.no_cache, globals.no_fetch) { // No local or remote cache - (true, true) => { - run.execute(ExecPlan::new(graph), None, &event_bus, &invocation_id) - .await - } + (true, true) => run.execute(ExecPlan::new(graph), None).await, // Both caches (false, false) => { let local_adapter = CacheBinProvider::new(graph, cache.clone()); @@ -162,8 +129,6 @@ pub async fn cmd_build_impl( run.execute( ExecPlan::new_with_bin_provider(graph, (local_adapter, remote_adapter)), Some(&remote_cache), - &event_bus, - &invocation_id, ) .await } @@ -174,21 +139,14 @@ pub async fn cmd_build_impl( run.execute( ExecPlan::new_with_bin_provider(graph, remote_adapter), Some(&remote_cache), - &event_bus, - &invocation_id, ) .await } // Only local cache (false, true) => { let local_adapter = CacheBinProvider::new(graph, cache.clone()); - run.execute( - ExecPlan::new_with_bin_provider(graph, local_adapter), - None, - &event_bus, - &invocation_id, - ) - .await + run.execute(ExecPlan::new_with_bin_provider(graph, local_adapter), None) + .await } }; @@ -196,11 +154,11 @@ pub async fn cmd_build_impl( let build_succeeded = build_success.is_ok(); let error_message = build_success.as_ref().err().map(|e| e.to_string()); - // Emit BuildFinished event - event_bus.emit(BuildEvent { - invocation_id: invocation_id.clone(), + // Emit BuildFinished event using global event bus + build_events::event_bus().emit(BuildEvent { + invocation_id: build_events::invocation_id().to_string(), event: Some(build_event::Event::BuildFinished(BuildFinished { - invocation_id: invocation_id.clone(), + invocation_id: build_events::invocation_id().to_string(), success: build_succeeded, timestamp_millis: current_millis(), error_message, @@ -261,10 +219,11 @@ pub async fn cmd_build_impl( } // Display build summary with spongebob URL - let spongebob_url = spongebob_invocation - .as_ref() - .map(|inv| inv.url().to_string()); - display_build_summary(graph, &cache, globals, &run, spongebob_url.as_deref()); + let spongebob_url = format!( + "https://dash.minimal.dev/invocations/{}", + build_events::invocation_id() + ); + display_build_summary(graph, &cache, globals, &run, Some(&spongebob_url)); Ok(()) } diff --git a/crates/minimal/src/cmd_patched_build.rs b/crates/minimal/src/cmd_patched_build.rs index 940b97f657..39d2ebae36 100644 --- a/crates/minimal/src/cmd_patched_build.rs +++ b/crates/minimal/src/cmd_patched_build.rs @@ -101,40 +101,13 @@ pub async fn cmd_patched_build(args: PatchedBuildArgs, globals: &GlobalArgs) -> .unwrap(); info!("Building package: {}", build.name); - let mut spongebob_client = spongebob::SpongeBob::new() - .await - .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; - - let spongebob_invocation = spongebob_client.create_invocation(); - let invocation_id = spongebob_invocation.invocation_id().to_string(); - - // Setup event bus and subscriber - use build_events::{BuildEventBus, BuildEventDispatcher}; - use build_events_proto::SpongeBobSubscriberV2; - - let event_bus = BuildEventBus::new(10000); - let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); - let subscriber = SpongeBobSubscriberV2::from_invocation(spongebob_invocation.clone()); - dispatcher.add_subscriber(Box::new(subscriber)); - - // Spawn dispatcher in background - tokio::spawn(async move { - dispatcher.run().await; - }); // Use package name as target ID for semantic meaning let target_id = build.name.clone(); - run_build( - &config, - out_dir.path(), - &event_bus, - &invocation_id, - output_base.clone(), - &target_id, - ) - .await - .map_err(|e| anyhow::anyhow!("Failed to build {}: {}", build.name, e))?; + run_build(&config, out_dir.path(), output_base.clone(), &target_id) + .await + .map_err(|e| anyhow::anyhow!("Failed to build {}: {}", build.name, e))?; out_dir .finalize(EntryMeta { diff --git a/crates/minimal/src/main.rs b/crates/minimal/src/main.rs index 9218bb21a8..8b10ff9519 100644 --- a/crates/minimal/src/main.rs +++ b/crates/minimal/src/main.rs @@ -362,6 +362,29 @@ async fn main() -> Result<()> { let cli = Cli::parse(); cli.global_args.path_config().ensure_directories()?; + // Initialize global event bus and SpongeBob integration + let mut spongebob_client = spongebob::SpongeBob::new() + .await + .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; + let spongebob_invocation = spongebob_client.create_invocation(); + let invocation_id = spongebob_invocation.invocation_id().to_string(); + + // Initialize the global event bus + build_events::initialize_global_event_bus(invocation_id); + + // Setup SpongeBob subscriber + use build_events::{BuildEventDispatcher}; + use build_events_proto::SpongeBobSubscriberV2; + + let mut dispatcher = BuildEventDispatcher::new(build_events::event_bus().subscribe()); + let subscriber = SpongeBobSubscriberV2::from_invocation(spongebob_invocation); + dispatcher.add_subscriber(Box::new(subscriber)); + + // Spawn dispatcher in background + tokio::spawn(async move { + dispatcher.run().await; + }); + let result = match cli.command { Command::Build(args) => cmd_build(args, &cli.global_args).await, Command::Check(args) => cmd_check(args, &cli.global_args), diff --git a/crates/minimal/src/run.rs b/crates/minimal/src/run.rs index be0555adfd..cf201fef84 100644 --- a/crates/minimal/src/run.rs +++ b/crates/minimal/src/run.rs @@ -462,8 +462,6 @@ impl<'a> Run<'a> { build: &BuildSpecRef, _full_build: bool, remote_cache: Option<&RemoteCache>, - event_bus: &build_events::BuildEventBus, - invocation_id: &str, ) -> Result> { let bsh = self.graph.spec_hash(build); let build = self.graph.get(build).unwrap(); @@ -544,8 +542,6 @@ impl<'a> Run<'a> { let _build_result = run_build( &config, out_dir.path(), - event_bus, - invocation_id, self.output_base.clone(), &target_id, ) @@ -564,8 +560,6 @@ impl<'a> Run<'a> { &mut self, plan: ExecPlan<'a, BP>, remote_cache: Option<&RemoteCache>, - event_bus: &build_events::BuildEventBus, - invocation_id: &str, ) -> Result<()> { // Execute builds in dependency order - each build runs in isolation // and can only access outputs from previously completed builds @@ -591,13 +585,8 @@ impl<'a> Run<'a> { s.spawn(move |_| { let _rt = tokio_runtime.enter(); - let result = futures::executor::block_on(self2.do_build( - &bsr, - full_build, - remote_cache, - event_bus, - invocation_id, - )); + let result = + futures::executor::block_on(self2.do_build(&bsr, full_build, remote_cache)); match result { Err(e) => { From 8a12d90c6ca2139076db8868af978a526d988320 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Tue, 14 Oct 2025 14:13:47 -0700 Subject: [PATCH 17/39] refactor --- Cargo.lock | 20 +++------ crates/build-events-proto/Cargo.toml | 4 +- crates/build-events-proto/src/service.rs | 1 + .../src/spongebob_subscriber_v2.rs | 4 +- crates/build-events/Cargo.toml | 2 +- .../build-events/src/subscribers/json_file.rs | 11 +++++ crates/build-events/src/subscribers/logger.rs | 10 +++++ crates/spongebob/Cargo.toml | 4 +- crates/spongebob/src/lib.rs | 43 +++++++++---------- 9 files changed, 56 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6976163d61..0eb1348641 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1455,12 +1455,6 @@ dependencies = [ "foldhash", ] -[[package]] -name = "hashbrown" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" - [[package]] name = "hashlink" version = "0.10.0" @@ -1811,7 +1805,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown 0.16.0", + "hashbrown 0.15.5", "serde", "serde_core", ] @@ -2229,9 +2223,9 @@ dependencies = [ [[package]] name = "minimal_spongebob_community_neoeinstein-prost" -version = "0.4.0-20251010205210-42ae8fd1517e.1" +version = "0.4.0-20251014210131-30c82017bd87.1" source = "sparse+https://buf.build/gen/cargo/" -checksum = "0d72206cc3cb5acbb8cfe6653c1ea5322c26f7a53cad76f8970a31f20a823bdf" +checksum = "ce549111413cc1c9772e4ea3b4a62dd25bf9e8d6eb0d918e0d519b58293e274e" dependencies = [ "prost 0.13.5", "prost-types 0.13.5", @@ -2239,9 +2233,9 @@ dependencies = [ [[package]] name = "minimal_spongebob_community_neoeinstein-tonic" -version = "0.4.1-20251010205210-42ae8fd1517e.1" +version = "0.4.1-20251014210131-30c82017bd87.1" source = "sparse+https://buf.build/gen/cargo/" -checksum = "46f7ecbf0ff7510877a847108be91c9541ca392fddd5336875c3c09a9ea302fb" +checksum = "36055716be965f6b27e2874c13529e8e6edc4b46c239165a3e91cd81878af22a" dependencies = [ "minimal_spongebob_community_neoeinstein-prost", "tonic 0.12.3", @@ -2821,7 +2815,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn", @@ -2834,7 +2828,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn", diff --git a/crates/build-events-proto/Cargo.toml b/crates/build-events-proto/Cargo.toml index f05ca55363..ba5e117934 100644 --- a/crates/build-events-proto/Cargo.toml +++ b/crates/build-events-proto/Cargo.toml @@ -19,8 +19,8 @@ tokio-stream = { version = "0.1", features = ["sync"] } spongebob = { path = "../spongebob", optional = true } # BSR proto dependencies for canonical spongebob.v1 types -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251010205210-42ae8fd1517e.1", registry = "buf" } -minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251010205210-42ae8fd1517e.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014210131-30c82017bd87.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251014210131-30c82017bd87.1", registry = "buf" } [features] default = [] diff --git a/crates/build-events-proto/src/service.rs b/crates/build-events-proto/src/service.rs index 9527ebbca8..3425f1cc79 100644 --- a/crates/build-events-proto/src/service.rs +++ b/crates/build-events-proto/src/service.rs @@ -163,6 +163,7 @@ impl BuildEventService for BuildEventServiceImpl { e.timestamp_millis } Some(proto::build_event::Event::BuildMetadata(_)) => 0, // Metadata events have no timestamp + Some(proto::build_event::Event::FileCreated(e)) => e.timestamp_millis, None => return None, }; diff --git a/crates/build-events-proto/src/spongebob_subscriber_v2.rs b/crates/build-events-proto/src/spongebob_subscriber_v2.rs index 4771a156a3..bf8e1159eb 100644 --- a/crates/build-events-proto/src/spongebob_subscriber_v2.rs +++ b/crates/build-events-proto/src/spongebob_subscriber_v2.rs @@ -109,7 +109,7 @@ impl BuildEventSubscriber for SpongeBobSubscriberV2 { // Then upload files if paths are available if let Some(stdout_path) = stdout_path { if let Ok(contents) = std::fs::read(stdout_path) { - if let Err(e) = invocation.upload_file(target_id, "stdout", contents).await { + if let Err(e) = invocation.publish_file_created_event(target_id, "stdout", contents).await { warn!("Failed to upload stdout for target {}: {}", target_id, e); } } else { @@ -119,7 +119,7 @@ impl BuildEventSubscriber for SpongeBobSubscriberV2 { if let Some(stderr_path) = stderr_path { if let Ok(contents) = std::fs::read(stderr_path) { - if let Err(e) = invocation.upload_file(target_id, "stderr", contents).await { + if let Err(e) = invocation.publish_file_created_event(target_id, "stderr", contents).await { warn!("Failed to upload stderr for target {}: {}", target_id, e); } } else { diff --git a/crates/build-events/Cargo.toml b/crates/build-events/Cargo.toml index 76ea6e0af6..c6aef2f6bc 100644 --- a/crates/build-events/Cargo.toml +++ b/crates/build-events/Cargo.toml @@ -15,7 +15,7 @@ tokio.workspace = true tracing.workspace = true # Proto types as single source of truth -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251010205210-42ae8fd1517e.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014210131-30c82017bd87.1", registry = "buf" } [dev-dependencies] serde_json.workspace = true diff --git a/crates/build-events/src/subscribers/json_file.rs b/crates/build-events/src/subscribers/json_file.rs index 38a396e86d..f1d37a2f85 100644 --- a/crates/build-events/src/subscribers/json_file.rs +++ b/crates/build-events/src/subscribers/json_file.rs @@ -155,6 +155,17 @@ fn event_to_json(event: &BuildEvent) -> serde_json::Value { } }) } + Some(build_event::Event::FileCreated(e)) => { + serde_json::json!({ + "type": "file_created", + "file_created": { + "target_id": e.target_id, + "name": e.name, + "size": e.contents.len(), + "timestamp_millis": e.timestamp_millis, + } + }) + } None => { serde_json::json!({ "type": "unknown", diff --git a/crates/build-events/src/subscribers/logger.rs b/crates/build-events/src/subscribers/logger.rs index 82c8e5c7c9..0cbff3485f 100644 --- a/crates/build-events/src/subscribers/logger.rs +++ b/crates/build-events/src/subscribers/logger.rs @@ -114,6 +114,16 @@ impl BuildEventSubscriber for LoggerSubscriber { "Build metadata" ); } + Some(build_event::Event::FileCreated(e)) => { + info!( + event = "file_created", + target_id = %e.target_id, + name = %e.name, + size = e.contents.len(), + timestamp_millis = e.timestamp_millis, + "File created" + ); + } None => { info!("Received event with no inner event type"); } diff --git a/crates/spongebob/Cargo.toml b/crates/spongebob/Cargo.toml index 6859528c59..f555463d55 100644 --- a/crates/spongebob/Cargo.toml +++ b/crates/spongebob/Cargo.toml @@ -5,8 +5,8 @@ edition.workspace = true publish.workspace = true [dependencies] -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251010205210-42ae8fd1517e.1", registry = "buf" } -minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251010205210-42ae8fd1517e.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014210131-30c82017bd87.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251014210131-30c82017bd87.1", registry = "buf" } thiserror.workspace = true tonic = { version = "0.12.3", features = ["tls", "tls-native-roots"] } tracing.workspace = true diff --git a/crates/spongebob/src/lib.rs b/crates/spongebob/src/lib.rs index 90eff1a7ca..e03ffea9bd 100644 --- a/crates/spongebob/src/lib.rs +++ b/crates/spongebob/src/lib.rs @@ -1,10 +1,7 @@ use minimal_spongebob_community_neoeinstein_prost::spongebob::v1::{ - BuildEvent, CreateFileRequest, File, PublishBuildEventRequest, -}; -use minimal_spongebob_community_neoeinstein_tonic::spongebob::v1::tonic::{ - build_event_service_client::BuildEventServiceClient, - result_storage_service_client::ResultStorageServiceClient, + build_event, BuildEvent, FileCreated, PublishBuildEventRequest, }; +use minimal_spongebob_community_neoeinstein_tonic::spongebob::v1::tonic::build_event_service_client::BuildEventServiceClient; use tonic::transport::Channel; use uuid::Uuid; @@ -23,13 +20,11 @@ pub type Result = std::result::Result; #[derive(Debug)] pub struct SpongeBob { build_event_service: BuildEventServiceClient, - result_storage_service: ResultStorageServiceClient, } #[derive(Debug, Clone)] pub struct SpongeBobInvocation { build_event_service: BuildEventServiceClient, - result_storage_service: ResultStorageServiceClient, invocation_id: String, url: String, } @@ -38,7 +33,6 @@ impl SpongeBobInvocation { /// Create a new invocation (no RPC needed - invocation created on first BuildStarted event) pub fn new( build_event_service: BuildEventServiceClient, - result_storage_service: ResultStorageServiceClient, invocation_id: String, ) -> Self { let url = format!( @@ -47,31 +41,37 @@ impl SpongeBobInvocation { ); Self { build_event_service, - result_storage_service, invocation_id, url, } } - /// Upload a file to a specific target within this invocation + /// Publish a FileCreated event to upload a file for a specific target #[tracing::instrument(skip(self, contents))] - pub async fn upload_file( + pub async fn publish_file_created_event( &mut self, target_id: &str, file_name: &str, contents: Vec, ) -> Result<()> { - let parent = format!("invocations/{}/targets/{}", self.invocation_id, target_id); - let request = CreateFileRequest { - parent, - file_id: file_name.to_string(), - file: Some(File { - name: file_name.to_string(), - contents: contents.into(), - }), + let timestamp_millis = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + + let file_created = FileCreated { + target_id: target_id.to_string(), + name: file_name.to_string(), + contents: contents.into(), + timestamp_millis, + }; + + let event = BuildEvent { + invocation_id: self.invocation_id.clone(), + event: Some(build_event::Event::FileCreated(file_created)), }; - self.result_storage_service.create_file(request).await?; + self.publish_build_event(event).await?; Ok(()) } @@ -106,11 +106,9 @@ impl SpongeBobInvocation { impl SpongeBob { pub async fn new() -> Result { let build_event_service = BuildEventServiceClient::connect(ENDPOINT).await?; - let result_storage_service = ResultStorageServiceClient::connect(ENDPOINT).await?; Ok(SpongeBob { build_event_service, - result_storage_service, }) } @@ -119,7 +117,6 @@ impl SpongeBob { let invocation_id = Uuid::new_v4().to_string(); SpongeBobInvocation::new( self.build_event_service.clone(), - self.result_storage_service.clone(), invocation_id, ) } From 2eb5a6403d25ac5ba4a8c11ac647272cfe29a82d Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Tue, 14 Oct 2025 14:55:28 -0700 Subject: [PATCH 18/39] new protocol --- Cargo.lock | 17 +- crates/build-events-proto/Cargo.toml | 4 +- crates/build-events-proto/src/service.rs | 34 ++-- .../src/spongebob_subscriber_v2.rs | 46 ++--- crates/build-events/Cargo.toml | 2 +- crates/build-events/src/events.rs | 3 - .../build-events/src/subscribers/json_file.rs | 7 +- crates/build-events/src/subscribers/logger.rs | 2 - crates/build-sandbox/src/executor.rs | 3 - crates/build-sandbox/src/executor_stub.rs | 4 +- crates/minimal/src/cmd_build.rs | 5 - crates/minimal/src/main.rs | 9 +- crates/spongebob/Cargo.toml | 7 +- crates/spongebob/src/lib.rs | 182 ++++++++++++------ 14 files changed, 185 insertions(+), 140 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0eb1348641..dcb6b34a1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2223,9 +2223,9 @@ dependencies = [ [[package]] name = "minimal_spongebob_community_neoeinstein-prost" -version = "0.4.0-20251014210131-30c82017bd87.1" +version = "0.4.0-20251014214004-4ceda9b7e17f.1" source = "sparse+https://buf.build/gen/cargo/" -checksum = "ce549111413cc1c9772e4ea3b4a62dd25bf9e8d6eb0d918e0d519b58293e274e" +checksum = "54c207aab4a172c7ffbe80866557cd86a2d2dc1b8672cff5a5165955dd578694" dependencies = [ "prost 0.13.5", "prost-types 0.13.5", @@ -2233,9 +2233,9 @@ dependencies = [ [[package]] name = "minimal_spongebob_community_neoeinstein-tonic" -version = "0.4.1-20251014210131-30c82017bd87.1" +version = "0.4.1-20251014214004-4ceda9b7e17f.1" source = "sparse+https://buf.build/gen/cargo/" -checksum = "36055716be965f6b27e2874c13529e8e6edc4b46c239165a3e91cd81878af22a" +checksum = "088a8ec4ad0d378cb9d24da98613b0f25c48ffd89fa28f7428a7be2ecce89946" dependencies = [ "minimal_spongebob_community_neoeinstein-prost", "tonic 0.12.3", @@ -2815,7 +2815,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -2828,7 +2828,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -3566,9 +3566,10 @@ dependencies = [ "minimal_spongebob_community_neoeinstein-prost", "minimal_spongebob_community_neoeinstein-tonic", "thiserror 2.0.17", + "tokio", + "tokio-stream", "tonic 0.12.3", "tracing", - "uuid", ] [[package]] @@ -4380,8 +4381,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ "getrandom 0.3.4", - "js-sys", - "wasm-bindgen", ] [[package]] diff --git a/crates/build-events-proto/Cargo.toml b/crates/build-events-proto/Cargo.toml index ba5e117934..cbad37c588 100644 --- a/crates/build-events-proto/Cargo.toml +++ b/crates/build-events-proto/Cargo.toml @@ -19,8 +19,8 @@ tokio-stream = { version = "0.1", features = ["sync"] } spongebob = { path = "../spongebob", optional = true } # BSR proto dependencies for canonical spongebob.v1 types -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014210131-30c82017bd87.1", registry = "buf" } -minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251014210131-30c82017bd87.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014214004-4ceda9b7e17f.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251014214004-4ceda9b7e17f.1", registry = "buf" } [features] default = [] diff --git a/crates/build-events-proto/src/service.rs b/crates/build-events-proto/src/service.rs index 3425f1cc79..e37c1b680b 100644 --- a/crates/build-events-proto/src/service.rs +++ b/crates/build-events-proto/src/service.rs @@ -131,22 +131,10 @@ impl BuildEventService for BuildEventServiceImpl { let filtered_stream = stream.filter_map(move |result| { match result { Ok(event) => { - // Apply invocation_id filter if specified - if let Some(ref filter_id) = invocation_filter { - // Check if event has matching invocation_id - let matches = match &event.event { - Some(proto::build_event::Event::BuildStarted(e)) => { - &e.invocation_id == filter_id - } - Some(proto::build_event::Event::BuildFinished(e)) => { - &e.invocation_id == filter_id - } - _ => true, // Other events pass through - }; - - if !matches { - return None; - } + // Note: invocation_id filtering is no longer supported at event level + // since events are implicitly associated with invocations via the stream + if invocation_filter.is_some() { + // Ignore filter for now - would need to track at stream level } // Apply timestamp filter if specified @@ -204,6 +192,20 @@ impl BuildEventService for BuildEventServiceImpl { name: String::new(), })) } + + type PublishBuildEventStreamStream = std::pin::Pin< + Box> + Send + 'static>, + >; + + async fn publish_build_event_stream( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + // Not implemented in this service - this is a client-side method + Err(tonic::Status::unimplemented( + "Bidirectional streaming not implemented in this build events service" + )) + } } #[cfg(test)] diff --git a/crates/build-events-proto/src/spongebob_subscriber_v2.rs b/crates/build-events-proto/src/spongebob_subscriber_v2.rs index bf8e1159eb..003253a5cf 100644 --- a/crates/build-events-proto/src/spongebob_subscriber_v2.rs +++ b/crates/build-events-proto/src/spongebob_subscriber_v2.rs @@ -10,7 +10,7 @@ use tracing::warn; /// Subscriber that publishes build events to Spongebob service /// /// This subscriber uses the workspace `spongebob` crate to connect to the -/// Spongebob service and publish build events. +/// Spongebob service and publish build events via bidirectional streaming. /// /// # Example /// ```no_run @@ -19,12 +19,11 @@ use tracing::warn; /// /// #[tokio::main] /// async fn main() -> Result<(), Box> { -/// // Create a spongebob client and invocation -/// let mut spongebob = spongebob::SpongeBob::new().await?; -/// let invocation = spongebob.create_invocation(); +/// // Create a spongebob client (opens stream and gets server-assigned ID) +/// let spongebob = spongebob::SpongeBob::new().await?; /// -/// // Create subscriber from the invocation -/// let subscriber = SpongeBobSubscriberV2::from_invocation(invocation); +/// // Create subscriber from the client +/// let subscriber = SpongeBobSubscriberV2::from_client(spongebob); /// /// // Use with BuildEventDispatcher /// // dispatcher.add_subscriber(Box::new(subscriber)); @@ -33,32 +32,31 @@ use tracing::warn; /// ``` #[cfg(feature = "spongebob-subscriber")] pub struct SpongeBobSubscriberV2 { - invocation: std::sync::Arc>, + spongebob: std::sync::Arc, /// Cache of file paths from BuildMetadata events, keyed by "{target_id}/stdout_path" etc. file_paths: std::sync::Arc>>, } #[cfg(feature = "spongebob-subscriber")] impl SpongeBobSubscriberV2 { - /// Create subscriber from an existing SpongeBobInvocation + /// Create subscriber from an existing SpongeBob client /// /// # Arguments - /// * `invocation` - An existing SpongeBobInvocation to publish events to + /// * `client` - An existing SpongeBob client with active streaming connection /// /// # Example /// ```no_run /// # #[cfg(feature = "spongebob-subscriber")] /// # use build_events_proto::SpongeBobSubscriberV2; /// # async fn example() -> Result<(), Box> { - /// let mut spongebob = spongebob::SpongeBob::new().await?; - /// let invocation = spongebob.create_invocation(); - /// let subscriber = SpongeBobSubscriberV2::from_invocation(invocation); + /// let spongebob = spongebob::SpongeBob::new().await?; + /// let subscriber = SpongeBobSubscriberV2::from_client(spongebob); /// # Ok(()) /// # } /// ``` - pub fn from_invocation(invocation: spongebob::SpongeBobInvocation) -> Self { + pub fn from_client(client: spongebob::SpongeBob) -> Self { Self { - invocation: std::sync::Arc::new(tokio::sync::Mutex::new(invocation)), + spongebob: std::sync::Arc::new(client), file_paths: std::sync::Arc::new(tokio::sync::Mutex::new( std::collections::HashMap::new(), )), @@ -93,12 +91,12 @@ impl BuildEventSubscriber for SpongeBobSubscriberV2 { let stdout_key = format!("{}/stdout_path", target_id); let stderr_key = format!("{}/stderr_path", target_id); - let stdout_path = file_paths.get(&stdout_key); - let stderr_path = file_paths.get(&stderr_key); + let stdout_path = file_paths.get(&stdout_key).cloned(); + let stderr_path = file_paths.get(&stderr_key).cloned(); + drop(file_paths); // First publish the TargetCompleted event - let mut invocation = self.invocation.lock().await; - if let Err(e) = invocation.publish_build_event(event.clone()).await { + if let Err(e) = self.spongebob.publish_build_event(event.clone()).await { warn!("Failed to publish TargetCompleted event to Spongebob: {}", e); return Err(SubscriberError::Custom(format!( "Failed to publish TargetCompleted event: {}", @@ -108,8 +106,8 @@ impl BuildEventSubscriber for SpongeBobSubscriberV2 { // Then upload files if paths are available if let Some(stdout_path) = stdout_path { - if let Ok(contents) = std::fs::read(stdout_path) { - if let Err(e) = invocation.publish_file_created_event(target_id, "stdout", contents).await { + if let Ok(contents) = std::fs::read(&stdout_path) { + if let Err(e) = self.spongebob.publish_file_created_event(target_id, "stdout", contents).await { warn!("Failed to upload stdout for target {}: {}", target_id, e); } } else { @@ -118,8 +116,8 @@ impl BuildEventSubscriber for SpongeBobSubscriberV2 { } if let Some(stderr_path) = stderr_path { - if let Ok(contents) = std::fs::read(stderr_path) { - if let Err(e) = invocation.publish_file_created_event(target_id, "stderr", contents).await { + if let Ok(contents) = std::fs::read(&stderr_path) { + if let Err(e) = self.spongebob.publish_file_created_event(target_id, "stderr", contents).await { warn!("Failed to upload stderr for target {}: {}", target_id, e); } } else { @@ -127,13 +125,11 @@ impl BuildEventSubscriber for SpongeBobSubscriberV2 { } } - drop(invocation); return Ok(()); } // For all other events, just publish them - let mut invocation = self.invocation.lock().await; - match invocation.publish_build_event(event.clone()).await { + match self.spongebob.publish_build_event(event.clone()).await { Ok(()) => { tracing::debug!("Successfully published build event to Spongebob"); Ok(()) diff --git a/crates/build-events/Cargo.toml b/crates/build-events/Cargo.toml index c6aef2f6bc..3adf2d6f81 100644 --- a/crates/build-events/Cargo.toml +++ b/crates/build-events/Cargo.toml @@ -15,7 +15,7 @@ tokio.workspace = true tracing.workspace = true # Proto types as single source of truth -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014210131-30c82017bd87.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014214004-4ceda9b7e17f.1", registry = "buf" } [dev-dependencies] serde_json.workspace = true diff --git a/crates/build-events/src/events.rs b/crates/build-events/src/events.rs index 59519e2601..d8496d5360 100644 --- a/crates/build-events/src/events.rs +++ b/crates/build-events/src/events.rs @@ -41,16 +41,13 @@ mod tests { #[test] fn test_event_construction() { let event = BuildEvent { - invocation_id: "test-123".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { - invocation_id: "test-123".to_string(), command: "build".to_string(), timestamp_millis: 1234567890, working_directory: "/tmp".to_string(), })), }; - assert_eq!(event.invocation_id, "test-123"); assert!(matches!( event.event, Some(build_event::Event::BuildStarted(_)) diff --git a/crates/build-events/src/subscribers/json_file.rs b/crates/build-events/src/subscribers/json_file.rs index f1d37a2f85..1cb6413812 100644 --- a/crates/build-events/src/subscribers/json_file.rs +++ b/crates/build-events/src/subscribers/json_file.rs @@ -74,7 +74,6 @@ fn event_to_json(event: &BuildEvent) -> serde_json::Value { serde_json::json!({ "type": "build_started", "build_started": { - "invocation_id": e.invocation_id, "command": e.command, "timestamp_millis": e.timestamp_millis, "working_directory": e.working_directory, @@ -85,7 +84,6 @@ fn event_to_json(event: &BuildEvent) -> serde_json::Value { serde_json::json!({ "type": "build_finished", "build_finished": { - "invocation_id": e.invocation_id, "success": e.success, "timestamp_millis": e.timestamp_millis, "error_message": e.error_message, @@ -173,10 +171,7 @@ fn event_to_json(event: &BuildEvent) -> serde_json::Value { } }; - serde_json::json!({ - "invocation_id": event.invocation_id, - "event": event_type_json, - }) + event_type_json } #[async_trait] diff --git a/crates/build-events/src/subscribers/logger.rs b/crates/build-events/src/subscribers/logger.rs index 0cbff3485f..8d5ca518d3 100644 --- a/crates/build-events/src/subscribers/logger.rs +++ b/crates/build-events/src/subscribers/logger.rs @@ -48,7 +48,6 @@ impl BuildEventSubscriber for LoggerSubscriber { Some(build_event::Event::BuildStarted(e)) => { info!( event = "build_started", - invocation_id = %e.invocation_id, command = %e.command, timestamp_millis = e.timestamp_millis, working_directory = %e.working_directory, @@ -58,7 +57,6 @@ impl BuildEventSubscriber for LoggerSubscriber { Some(build_event::Event::BuildFinished(e)) => { info!( event = "build_finished", - invocation_id = %e.invocation_id, success = e.success, timestamp_millis = e.timestamp_millis, error_message = ?e.error_message, diff --git a/crates/build-sandbox/src/executor.rs b/crates/build-sandbox/src/executor.rs index f34ac6827e..b10ac36f5c 100644 --- a/crates/build-sandbox/src/executor.rs +++ b/crates/build-sandbox/src/executor.rs @@ -80,7 +80,6 @@ impl BuildExecutor { // Emit TargetStarted event using global event bus build_events::event_bus().emit(BuildEvent { - invocation_id: build_events::invocation_id().to_string(), event: Some(build_event::Event::TargetStarted(TargetStarted { target_id: target_id.to_string(), label: config.name.clone(), @@ -98,7 +97,6 @@ impl BuildExecutor { }; build_events::event_bus().emit(BuildEvent { - invocation_id: build_events::invocation_id().to_string(), event: Some(build_event::Event::TargetCompleted(TargetCompleted { target_id: target_id.to_string(), label: config.name.clone(), @@ -128,7 +126,6 @@ impl BuildExecutor { ); build_events::event_bus().emit(BuildEvent { - invocation_id: build_events::invocation_id().to_string(), event: Some(build_event::Event::BuildMetadata(BuildMetadata { metadata: file_metadata, })), diff --git a/crates/build-sandbox/src/executor_stub.rs b/crates/build-sandbox/src/executor_stub.rs index 234416bdee..61f92086a7 100644 --- a/crates/build-sandbox/src/executor_stub.rs +++ b/crates/build-sandbox/src/executor_stub.rs @@ -16,9 +16,8 @@ impl BuildExecutor { pub async fn execute( &self, _: &BuildConfig, - _: &mut Option, _: &str, - ) -> Result<(i32, Option)> { + ) -> Result { todo!() } @@ -37,7 +36,6 @@ impl BuildExecutor { async fn execute_in_container( &self, _: &BuildConfig, - _: &mut Option, _: &str, ) -> Result<()> { todo!() diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index 309c637a96..00f023cb3d 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -66,9 +66,7 @@ pub async fn cmd_build_impl( // Emit BuildStarted event using global event bus build_events::event_bus().emit(BuildEvent { - invocation_id: build_events::invocation_id().to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { - invocation_id: build_events::invocation_id().to_string(), command, timestamp_millis: current_millis(), working_directory, @@ -113,7 +111,6 @@ pub async fn cmd_build_impl( // Emit BuildMetadata event if we collected any metadata (using global event bus) if !metadata.is_empty() { build_events::event_bus().emit(BuildEvent { - invocation_id: build_events::invocation_id().to_string(), event: Some(build_event::Event::BuildMetadata(BuildMetadata { metadata })), }); } @@ -156,9 +153,7 @@ pub async fn cmd_build_impl( // Emit BuildFinished event using global event bus build_events::event_bus().emit(BuildEvent { - invocation_id: build_events::invocation_id().to_string(), event: Some(build_event::Event::BuildFinished(BuildFinished { - invocation_id: build_events::invocation_id().to_string(), success: build_succeeded, timestamp_millis: current_millis(), error_message, diff --git a/crates/minimal/src/main.rs b/crates/minimal/src/main.rs index 8b10ff9519..9a52d02d24 100644 --- a/crates/minimal/src/main.rs +++ b/crates/minimal/src/main.rs @@ -363,21 +363,20 @@ async fn main() -> Result<()> { cli.global_args.path_config().ensure_directories()?; // Initialize global event bus and SpongeBob integration - let mut spongebob_client = spongebob::SpongeBob::new() + let spongebob = spongebob::SpongeBob::new() .await .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; - let spongebob_invocation = spongebob_client.create_invocation(); - let invocation_id = spongebob_invocation.invocation_id().to_string(); + let invocation_id = spongebob.invocation_id().to_string(); // Initialize the global event bus build_events::initialize_global_event_bus(invocation_id); // Setup SpongeBob subscriber - use build_events::{BuildEventDispatcher}; + use build_events::BuildEventDispatcher; use build_events_proto::SpongeBobSubscriberV2; let mut dispatcher = BuildEventDispatcher::new(build_events::event_bus().subscribe()); - let subscriber = SpongeBobSubscriberV2::from_invocation(spongebob_invocation); + let subscriber = SpongeBobSubscriberV2::from_client(spongebob); dispatcher.add_subscriber(Box::new(subscriber)); // Spawn dispatcher in background diff --git a/crates/spongebob/Cargo.toml b/crates/spongebob/Cargo.toml index f555463d55..4c55232238 100644 --- a/crates/spongebob/Cargo.toml +++ b/crates/spongebob/Cargo.toml @@ -5,9 +5,10 @@ edition.workspace = true publish.workspace = true [dependencies] -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014210131-30c82017bd87.1", registry = "buf" } -minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251014210131-30c82017bd87.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014214004-4ceda9b7e17f.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251014214004-4ceda9b7e17f.1", registry = "buf" } thiserror.workspace = true +tokio = { version = "1", features = ["sync"] } +tokio-stream = "0.1" tonic = { version = "0.12.3", features = ["tls", "tls-native-roots"] } tracing.workspace = true -uuid.workspace = true diff --git a/crates/spongebob/src/lib.rs b/crates/spongebob/src/lib.rs index e03ffea9bd..c3b65c934d 100644 --- a/crates/spongebob/src/lib.rs +++ b/crates/spongebob/src/lib.rs @@ -1,9 +1,14 @@ use minimal_spongebob_community_neoeinstein_prost::spongebob::v1::{ - build_event, BuildEvent, FileCreated, PublishBuildEventRequest, + build_event, BuildEvent, FileCreated, OrderedBuildEvent, PublishBuildEventStreamRequest, + PublishBuildEventStreamResponse, }; use minimal_spongebob_community_neoeinstein_tonic::spongebob::v1::tonic::build_event_service_client::BuildEventServiceClient; -use tonic::transport::Channel; -use uuid::Uuid; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::sync::Arc; +use tokio::sync::mpsc; +use tokio::sync::Mutex; +use tokio_stream::wrappers::ReceiverStream; +use tonic::Streaming; const ENDPOINT: &str = "https://spongebob.minimal.farm"; @@ -13,43 +18,96 @@ pub enum SpongeBobError { Connection(#[from] tonic::transport::Error), #[error("SpongeBob service error: {0}")] Service(#[from] tonic::Status), + #[error("Stream error: {0}")] + Stream(String), } pub type Result = std::result::Result; -#[derive(Debug)] -pub struct SpongeBob { - build_event_service: BuildEventServiceClient, -} - +/// SpongeBob client with server-assigned invocation ID +/// +/// Creates a bidirectional stream on initialization and receives +/// a server-assigned invocation ID. All events are published through +/// this stream with monotonically increasing sequence numbers. #[derive(Debug, Clone)] -pub struct SpongeBobInvocation { - build_event_service: BuildEventServiceClient, +pub struct SpongeBob { + request_tx: Arc>>, + response_stream: Arc>>, invocation_id: String, url: String, + sequence_number: Arc, } -impl SpongeBobInvocation { - /// Create a new invocation (no RPC needed - invocation created on first BuildStarted event) - pub fn new( - build_event_service: BuildEventServiceClient, - invocation_id: String, - ) -> Self { - let url = format!( - "{}/invocations/{}", - "https://dash.minimal.dev", invocation_id - ); - Self { - build_event_service, +impl SpongeBob { + /// Create a new SpongeBob client and open streaming connection + /// + /// This immediately connects to the server and opens a bidirectional + /// stream. The server assigns an invocation ID which is returned in + /// the first response. + /// + /// # Example + /// ```no_run + /// # async fn example() -> Result<(), Box> { + /// let spongebob = spongebob::SpongeBob::new().await?; + /// println!("Invocation URL: {}", spongebob.url()); + /// # Ok(()) + /// # } + /// ``` + #[tracing::instrument] + pub async fn new() -> Result { + // Connect to service + let mut client = BuildEventServiceClient::connect(ENDPOINT).await?; + + // Create channel for bidirectional streaming + let (request_tx, request_rx) = mpsc::channel(100); + + // Wrap receiver in a stream + let request_stream = ReceiverStream::new(request_rx); + + // Open bidirectional stream + let response = client.publish_build_event_stream(request_stream).await?; + let mut response_stream = response.into_inner(); + + // Send first request with sequence number 1 to get server-assigned ID + request_tx + .send(PublishBuildEventStreamRequest { + ordered_event: Some(OrderedBuildEvent { + sequence_number: 1, + stream_id: None, + event: None, + }), + }) + .await + .map_err(|e| SpongeBobError::Stream(format!("Failed to send initial request: {}", e)))?; + + // Wait for first response with server-assigned invocation ID + let first_response = response_stream + .message() + .await? + .ok_or_else(|| SpongeBobError::Stream("No response from server".to_string()))?; + + let invocation_id = first_response + .stream_id + .ok_or_else(|| SpongeBobError::Stream("Server did not provide stream ID".to_string()))? + .invocation_id; + + let url = format!("https://dash.minimal.dev/invocations/{}", invocation_id); + + tracing::info!(invocation_id = %invocation_id, "Received server-assigned invocation ID"); + + Ok(Self { + request_tx: Arc::new(Mutex::new(request_tx)), + response_stream: Arc::new(Mutex::new(response_stream)), invocation_id, url, - } + sequence_number: Arc::new(AtomicI64::new(2)), // Start at 2 (we used 1 for initial request) + }) } /// Publish a FileCreated event to upload a file for a specific target #[tracing::instrument(skip(self, contents))] pub async fn publish_file_created_event( - &mut self, + &self, target_id: &str, file_name: &str, contents: Vec, @@ -67,57 +125,67 @@ impl SpongeBobInvocation { }; let event = BuildEvent { - invocation_id: self.invocation_id.clone(), event: Some(build_event::Event::FileCreated(file_created)), }; - self.publish_build_event(event).await?; - Ok(()) + self.publish_build_event(event).await } - /// Publish a build event to Spongebob - /// Invocation is auto-created on first BuildStarted event + /// Publish a build event to Spongebob via the streaming connection #[tracing::instrument(skip(self, event))] - pub async fn publish_build_event(&mut self, event: BuildEvent) -> Result<()> { - let request = PublishBuildEventRequest { - event: Some(event), - request_id: String::new(), // Optional idempotency key + pub async fn publish_build_event(&self, event: BuildEvent) -> Result<()> { + let seq = self.sequence_number.fetch_add(1, Ordering::SeqCst); + + // Send event with sequence number + let request = PublishBuildEventStreamRequest { + ordered_event: Some(OrderedBuildEvent { + sequence_number: seq, + stream_id: None, // Only needed on first message + event: Some(event), + }), }; - self.build_event_service - .publish_build_event(request) - .await?; + let request_tx = self.request_tx.lock().await; + request_tx + .send(request) + .await + .map_err(|e| SpongeBobError::Stream(format!("Failed to send event: {}", e)))?; + + // Wait for acknowledgment + let mut response_stream = self.response_stream.lock().await; + let _ack = response_stream + .message() + .await? + .ok_or_else(|| SpongeBobError::Stream("No acknowledgment from server".to_string()))?; + + tracing::debug!(sequence_number = seq, "Event acknowledged by server"); + Ok(()) } - pub fn url(&self) -> &str { - &self.url + /// Close the stream (drops the connection) + /// + /// If BuildFinished was not sent, the server will mark the invocation + /// as terminated. + pub async fn close(self) -> Result<()> { + // Drop the request sender which will close the stream + drop(self.request_tx); + drop(self.response_stream); + Ok(()) } + /// Get the server-assigned invocation ID pub fn invocation_id(&self) -> &str { &self.invocation_id } - pub fn resource_name(&self) -> String { - format!("invocations/{}", self.invocation_id) - } -} - -impl SpongeBob { - pub async fn new() -> Result { - let build_event_service = BuildEventServiceClient::connect(ENDPOINT).await?; - - Ok(SpongeBob { - build_event_service, - }) + /// Get the dashboard URL for this invocation + pub fn url(&self) -> &str { + &self.url } - /// Create a new invocation handle (no RPC - invocation created on first BuildStarted event) - pub fn create_invocation(&mut self) -> SpongeBobInvocation { - let invocation_id = Uuid::new_v4().to_string(); - SpongeBobInvocation::new( - self.build_event_service.clone(), - invocation_id, - ) + /// Get the resource name for this invocation + pub fn resource_name(&self) -> String { + format!("invocations/{}", self.invocation_id) } } From e2223f0ecc3615afe87a46983040484a847ef84a Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Wed, 15 Oct 2025 11:29:59 -0700 Subject: [PATCH 19/39] merge crates --- Cargo.lock | 16 -- Cargo.toml | 1 - crates/build-events-proto/Cargo.toml | 13 +- .../src/client.rs} | 121 ++++++--- crates/build-events-proto/src/lib.rs | 95 ++----- crates/build-events-proto/src/service.rs | 243 ------------------ .../src/spongebob_subscriber_v2.rs | 30 +-- crates/build-events-proto/src/subscriber.rs | 158 ------------ crates/minimal/Cargo.toml | 3 +- crates/minimal/src/cmd_build.rs | 3 +- crates/minimal/src/main.rs | 2 +- crates/spongebob/Cargo.toml | 14 - 12 files changed, 125 insertions(+), 574 deletions(-) rename crates/{spongebob/src/lib.rs => build-events-proto/src/client.rs} (56%) delete mode 100644 crates/build-events-proto/src/service.rs delete mode 100644 crates/build-events-proto/src/subscriber.rs delete mode 100644 crates/spongebob/Cargo.toml diff --git a/Cargo.lock b/Cargo.lock index dcb6b34a1e..450baeae49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,7 +364,6 @@ dependencies = [ "build-events", "minimal_spongebob_community_neoeinstein-prost", "minimal_spongebob_community_neoeinstein-tonic", - "spongebob", "tempfile", "thiserror 2.0.17", "tokio", @@ -2210,7 +2209,6 @@ dependencies = [ "sha2", "shlex", "size", - "spongebob", "tar", "tempfile", "tokio", @@ -3559,19 +3557,6 @@ dependencies = [ "windows-sys 0.60.2", ] -[[package]] -name = "spongebob" -version = "0.1.0" -dependencies = [ - "minimal_spongebob_community_neoeinstein-prost", - "minimal_spongebob_community_neoeinstein-tonic", - "thiserror 2.0.17", - "tokio", - "tokio-stream", - "tonic 0.12.3", - "tracing", -] - [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -3897,7 +3882,6 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", - "tokio-util", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d8dbf097d4..9add4bd8af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,6 @@ members = [ "crates/common", "crates/graph", "crates/minimal", - "crates/spongebob", ] resolver = "3" diff --git a/crates/build-events-proto/Cargo.toml b/crates/build-events-proto/Cargo.toml index cbad37c588..a9aa043547 100644 --- a/crates/build-events-proto/Cargo.toml +++ b/crates/build-events-proto/Cargo.toml @@ -8,24 +8,17 @@ publish.workspace = true [dependencies] build-events.workspace = true -tonic = "0.12.3" -tokio.workspace = true +tonic = { version = "0.12.3", features = ["tls", "tls-native-roots"] } +tokio = { workspace = true, features = ["sync"] } +tokio-stream = "0.1" thiserror.workspace = true tracing.workspace = true async-trait = "0.1" -tokio-stream = { version = "0.1", features = ["sync"] } - -# Use workspace spongebob crate instead of direct BSR imports -spongebob = { path = "../spongebob", optional = true } # BSR proto dependencies for canonical spongebob.v1 types minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014214004-4ceda9b7e17f.1", registry = "buf" } minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251014214004-4ceda9b7e17f.1", registry = "buf" } -[features] -default = [] -spongebob-subscriber = ["spongebob"] - [dev-dependencies] tempfile.workspace = true tracing-subscriber.workspace = true diff --git a/crates/spongebob/src/lib.rs b/crates/build-events-proto/src/client.rs similarity index 56% rename from crates/spongebob/src/lib.rs rename to crates/build-events-proto/src/client.rs index c3b65c934d..b497603a19 100644 --- a/crates/spongebob/src/lib.rs +++ b/crates/build-events-proto/src/client.rs @@ -1,14 +1,14 @@ -use minimal_spongebob_community_neoeinstein_prost::spongebob::v1::{ - build_event, BuildEvent, FileCreated, OrderedBuildEvent, PublishBuildEventStreamRequest, - PublishBuildEventStreamResponse, +//! SpongeBob gRPC client for streaming build events + +use crate::proto::{ + build_event, BuildEvent, BuildStarted, FileCreated, OrderedBuildEvent, PublishBuildEventStreamRequest, + build_event_service_client::BuildEventServiceClient, }; -use minimal_spongebob_community_neoeinstein_tonic::spongebob::v1::tonic::build_event_service_client::BuildEventServiceClient; use std::sync::atomic::{AtomicI64, Ordering}; use std::sync::Arc; use tokio::sync::mpsc; use tokio::sync::Mutex; use tokio_stream::wrappers::ReceiverStream; -use tonic::Streaming; const ENDPOINT: &str = "https://spongebob.minimal.farm"; @@ -29,10 +29,12 @@ pub type Result = std::result::Result; /// Creates a bidirectional stream on initialization and receives /// a server-assigned invocation ID. All events are published through /// this stream with monotonically increasing sequence numbers. +/// +/// The client spawns a background task to continuously read from the +/// response stream, allowing non-blocking event publishing. #[derive(Debug, Clone)] pub struct SpongeBob { request_tx: Arc>>, - response_stream: Arc>>, invocation_id: String, url: String, sequence_number: Arc, @@ -46,9 +48,9 @@ impl SpongeBob { /// the first response. /// /// # Example - /// ```no_run + /// ```ignore /// # async fn example() -> Result<(), Box> { - /// let spongebob = spongebob::SpongeBob::new().await?; + /// let spongebob = build_events_proto::SpongeBob::new().await?; /// println!("Invocation URL: {}", spongebob.url()); /// # Ok(()) /// # } @@ -61,46 +63,95 @@ impl SpongeBob { // Create channel for bidirectional streaming let (request_tx, request_rx) = mpsc::channel(100); - // Wrap receiver in a stream - let request_stream = ReceiverStream::new(request_rx); - - // Open bidirectional stream - let response = client.publish_build_event_stream(request_stream).await?; - let mut response_stream = response.into_inner(); + // Send initial BuildStarted event to initiate handshake + // Server will respond with assigned invocation ID + let timestamp_millis = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64; - // Send first request with sequence number 1 to get server-assigned ID request_tx .send(PublishBuildEventStreamRequest { ordered_event: Some(OrderedBuildEvent { sequence_number: 1, - stream_id: None, - event: None, + stream_id: None, // Server will assign + event: Some(BuildEvent { + event: Some(build_event::Event::BuildStarted(BuildStarted { + command: std::env::args().collect::>().join(" "), + timestamp_millis, + working_directory: std::env::current_dir() + .ok() + .and_then(|p| p.to_str().map(|s| s.to_string())) + .unwrap_or_default(), + })), + }), }), }) .await - .map_err(|e| SpongeBobError::Stream(format!("Failed to send initial request: {}", e)))?; + .map_err(|e| SpongeBobError::Stream(format!("Failed to queue initial event: {}", e)))?; + + // Wrap receiver in a stream (initial BuildStarted is already queued) + let request_stream = ReceiverStream::new(request_rx); - // Wait for first response with server-assigned invocation ID - let first_response = response_stream - .message() - .await? - .ok_or_else(|| SpongeBobError::Stream("No response from server".to_string()))?; + // Open bidirectional stream - this will immediately have data to send + let response = client.publish_build_event_stream(request_stream).await?; + let mut response_stream = response.into_inner(); + + // Create oneshot channel for first response + let (first_response_tx, first_response_rx) = tokio::sync::oneshot::channel(); + + // Spawn background task to continuously read responses + // This task reads responses and sends the first one to the oneshot channel + tokio::spawn(async move { + let mut first_response_tx = Some(first_response_tx); + + loop { + match response_stream.message().await { + Ok(Some(response)) => { + // Send first response to oneshot channel + if let Some(tx) = first_response_tx.take() { + let _ = tx.send(response); + } else { + // Log acknowledgment for debugging + tracing::debug!( + sequence_number = response.sequence_number, + "Received acknowledgment from server" + ); + } + } + Ok(None) => { + // Stream closed by server + tracing::info!("Response stream closed by server"); + break; + } + Err(e) => { + // Log error but don't crash - allow graceful degradation + tracing::warn!("Error reading from response stream: {}", e); + break; + } + } + } + }); + + // Wait for first response from background task + let first_response = first_response_rx + .await + .map_err(|_| SpongeBobError::Stream("Background task died before first response".to_string()))?; let invocation_id = first_response .stream_id .ok_or_else(|| SpongeBobError::Stream("Server did not provide stream ID".to_string()))? .invocation_id; - let url = format!("https://dash.minimal.dev/invocations/{}", invocation_id); + let url = format!("https://dash.minimal.farm/invocations/{}", invocation_id); tracing::info!(invocation_id = %invocation_id, "Received server-assigned invocation ID"); Ok(Self { request_tx: Arc::new(Mutex::new(request_tx)), - response_stream: Arc::new(Mutex::new(response_stream)), invocation_id, url, - sequence_number: Arc::new(AtomicI64::new(2)), // Start at 2 (we used 1 for initial request) + sequence_number: Arc::new(AtomicI64::new(2)), // Start at 2 (we used 1 for BuildStarted) }) } @@ -132,6 +183,10 @@ impl SpongeBob { } /// Publish a build event to Spongebob via the streaming connection + /// + /// This method is non-blocking - it sends the event to the stream without + /// waiting for acknowledgment. Acknowledgments are processed asynchronously + /// by a background task spawned in `new()`. #[tracing::instrument(skip(self, event))] pub async fn publish_build_event(&self, event: BuildEvent) -> Result<()> { let seq = self.sequence_number.fetch_add(1, Ordering::SeqCst); @@ -151,14 +206,7 @@ impl SpongeBob { .await .map_err(|e| SpongeBobError::Stream(format!("Failed to send event: {}", e)))?; - // Wait for acknowledgment - let mut response_stream = self.response_stream.lock().await; - let _ack = response_stream - .message() - .await? - .ok_or_else(|| SpongeBobError::Stream("No acknowledgment from server".to_string()))?; - - tracing::debug!(sequence_number = seq, "Event acknowledged by server"); + tracing::debug!(sequence_number = seq, "Event sent to stream"); Ok(()) } @@ -166,11 +214,12 @@ impl SpongeBob { /// Close the stream (drops the connection) /// /// If BuildFinished was not sent, the server will mark the invocation - /// as terminated. + /// as terminated. The background task reading responses will detect + /// the closed stream and terminate gracefully. pub async fn close(self) -> Result<()> { // Drop the request sender which will close the stream + // The background task will detect the closed stream and terminate drop(self.request_tx); - drop(self.response_stream); Ok(()) } diff --git a/crates/build-events-proto/src/lib.rs b/crates/build-events-proto/src/lib.rs index 31009fe4b7..3d000486e5 100644 --- a/crates/build-events-proto/src/lib.rs +++ b/crates/build-events-proto/src/lib.rs @@ -1,89 +1,49 @@ //! Build Events Protobuf Layer //! -//! This crate provides protobuf definitions and gRPC streaming support for -//! the `build-events` crate. It enables remote observability of builds via -//! gRPC streaming. +//! This crate provides protobuf definitions and SpongeBob client integration for +//! the `build-events` crate. It enables remote observability of builds by sending +//! events to the SpongeBob service via gRPC streaming. //! //! # Overview //! //! This crate extends the pure Rust `build-events` crate with: -//! - **Protobuf message definitions** for all build event types -//! - **Type conversions** between Rust and protobuf types -//! - **GrpcStreamSubscriber** for converting events to proto and streaming -//! - **BuildEventService** gRPC service implementation +//! - **Protobuf message definitions** for all build event types (via BSR) +//! - **SpongeBob gRPC client** for streaming build events to spongebob.minimal.farm +//! - **SpongeBobSubscriberV2** for forwarding events to SpongeBob service //! //! # Architecture //! //! ```text -//! Build Events (Rust) → Conversion → Proto Events → gRPC Stream → Clients +//! Build Events → BuildEventDispatcher → SpongeBobSubscriberV2 → SpongeBob client → spongebob.minimal.farm //! ``` //! -//! # Example: gRPC Streaming +//! # Example: Sending Events to SpongeBob //! -//! ## Server Side -//! -//! ```no_run +//! ```ignore //! use build_events::{BuildEventBus, BuildEventDispatcher}; -//! use build_events_proto::{ -//! GrpcStreamSubscriber, -//! BuildEventServiceImpl, -//! proto::build_event_service_server::BuildEventServiceServer, -//! }; -//! use tonic::transport::Server; +//! use build_events_proto::{SpongeBob, SpongeBobSubscriberV2}; //! //! #[tokio::main] //! async fn main() -> Result<(), Box> { -//! // Create event bus -//! let event_bus = BuildEventBus::new(10000); +//! // Create SpongeBob client (opens bidirectional stream) +//! let spongebob = SpongeBob::new().await?; +//! let invocation_id = spongebob.invocation_id().to_string(); //! -//! // Create gRPC service -//! let service = BuildEventServiceImpl::new(1000); -//! let sender = service.sender(); +//! // Initialize event bus +//! let event_bus = BuildEventBus::new(10000); //! -//! // Setup dispatcher with a subscriber that feeds the gRPC service +//! // Setup dispatcher with SpongeBob subscriber //! let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); -//! let subscriber = GrpcStreamSubscriber::new(100); +//! let subscriber = SpongeBobSubscriberV2::from_client(spongebob); //! dispatcher.add_subscriber(Box::new(subscriber)); //! -//! // Start dispatcher +//! // Start dispatcher in background //! tokio::spawn(async move { //! dispatcher.run().await; //! }); //! -//! // Start gRPC server -//! Server::builder() -//! .add_service(BuildEventServiceServer::new(service)) -//! .serve("[::1]:50051".parse()?) -//! .await?; -//! -//! Ok(()) -//! } -//! ``` -//! -//! ## Client Side -//! -//! ```no_run -//! use build_events_proto::proto::{ -//! build_event_service_client::BuildEventServiceClient, -//! StreamBuildEventsRequest, -//! }; -//! use tokio_stream::StreamExt; -//! -//! #[tokio::main] -//! async fn main() -> Result<(), Box> { -//! let mut client = BuildEventServiceClient::connect("http://[::1]:50051").await?; -//! -//! let request = StreamBuildEventsRequest { -//! invocation_id: None, -//! start_timestamp_millis: None, -//! }; -//! -//! let mut stream = client.stream_build_events(request).await?.into_inner(); -//! -//! while let Some(event) = stream.next().await { -//! let event = event?; -//! println!("Received event: {:?}", event); -//! } +//! // Events emitted to event_bus are now forwarded to SpongeBob +//! // ... perform builds ... //! //! Ok(()) //! } @@ -91,17 +51,14 @@ //! //! # Type Usage //! -//! Since build-events now uses proto types directly, no conversion is needed: +//! Since build-events uses proto types directly from BSR, no conversion is needed: //! //! ``` //! use build_events::events::{build_event, BuildEvent, BuildStarted, current_millis}; -//! use build_events_proto::proto; //! //! // Create proto event (build-events types ARE proto types) //! let event = BuildEvent { -//! invocation_id: "test-123".to_string(), //! event: Some(build_event::Event::BuildStarted(BuildStarted { -//! invocation_id: "test-123".to_string(), //! command: "build".to_string(), //! timestamp_millis: current_millis(), //! working_directory: "/tmp".to_string(), @@ -120,15 +77,9 @@ pub mod proto { }; } -pub mod service; -pub mod subscriber; - -#[cfg(feature = "spongebob-subscriber")] +pub mod client; pub mod spongebob_subscriber_v2; // Re-export main types for convenience -pub use service::BuildEventServiceImpl; -pub use subscriber::GrpcStreamSubscriber; - -#[cfg(feature = "spongebob-subscriber")] +pub use client::{SpongeBob, SpongeBobError, Result}; pub use spongebob_subscriber_v2::SpongeBobSubscriberV2; diff --git a/crates/build-events-proto/src/service.rs b/crates/build-events-proto/src/service.rs deleted file mode 100644 index e37c1b680b..0000000000 --- a/crates/build-events-proto/src/service.rs +++ /dev/null @@ -1,243 +0,0 @@ -//! gRPC service implementation for build events -//! -//! Provides a [`BuildEventServiceImpl`] that implements the gRPC service -//! defined in the protobuf schema. - -use tokio::sync::broadcast; -use tokio_stream::wrappers::BroadcastStream; -use tokio_stream::{Stream, StreamExt}; -use tracing::{info, warn}; - -use crate::proto; -use crate::proto::build_event_service_server::BuildEventService; - -/// Implementation of the BuildEventService gRPC service -/// -/// This service provides server-side streaming of build events. It uses -/// a broadcast channel to allow multiple clients to subscribe to the same -/// event stream concurrently. -/// -/// # Example -/// ```no_run -/// use build_events_proto::{BuildEventServiceImpl, proto::build_event_service_server::BuildEventServiceServer}; -/// use tonic::transport::Server; -/// -/// #[tokio::main] -/// async fn main() -> Result<(), Box> { -/// let service = BuildEventServiceImpl::new(1000); -/// -/// Server::builder() -/// .add_service(BuildEventServiceServer::new(service)) -/// .serve("[::1]:50051".parse()?) -/// .await?; -/// -/// Ok(()) -/// } -/// ``` -pub struct BuildEventServiceImpl { - event_sender: broadcast::Sender, -} - -impl BuildEventServiceImpl { - /// Create a new build event service with the specified channel capacity - /// - /// # Arguments - /// * `capacity` - Broadcast channel capacity - /// - /// # Example - /// ``` - /// use build_events_proto::BuildEventServiceImpl; - /// - /// let service = BuildEventServiceImpl::new(1000); - /// ``` - pub fn new(capacity: usize) -> Self { - let (sender, _) = broadcast::channel(capacity); - Self { - event_sender: sender, - } - } - - /// Publish an event to all subscribers - /// - /// This method allows the build system to publish events that will - /// be streamed to all connected gRPC clients. - /// - /// # Arguments - /// * `event` - The protobuf event to publish - /// - /// # Example - /// ``` - /// use build_events_proto::{BuildEventServiceImpl, proto}; - /// - /// let service = BuildEventServiceImpl::new(1000); - /// let event = proto::BuildEvent { - /// invocation_id: String::new(), - /// event: None - /// }; - /// service.publish(event); - /// ``` - pub fn publish(&self, event: proto::BuildEvent) { - // Broadcast to all subscribers - // If there are no subscribers, this is a no-op - let subscriber_count = self.event_sender.receiver_count(); - if subscriber_count == 0 { - return; - } - - if let Err(e) = self.event_sender.send(event) { - warn!("Failed to broadcast event: no receivers ({})", e); - } - } - - /// Get a sender for publishing events - /// - /// This is useful for integrating with the build system's event bus. - pub fn sender(&self) -> broadcast::Sender { - self.event_sender.clone() - } - - /// Get the current number of active subscribers - pub fn subscriber_count(&self) -> usize { - self.event_sender.receiver_count() - } -} - -#[tonic::async_trait] -impl BuildEventService for BuildEventServiceImpl { - type StreamBuildEventsStream = std::pin::Pin< - Box> + Send + 'static>, - >; - - async fn stream_build_events( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let req = request.into_inner(); - - info!( - "New event stream subscription: invocation_id={:?}, start_timestamp={:?}", - req.invocation_id, req.start_timestamp_millis - ); - - // Subscribe to the broadcast channel - let receiver = self.event_sender.subscribe(); - let stream = BroadcastStream::new(receiver); - - // Apply filters if specified - let invocation_filter = req.invocation_id.clone(); - let timestamp_filter = req.start_timestamp_millis; - - // Wrap the stream with filtering logic and error mapping - let filtered_stream = stream.filter_map(move |result| { - match result { - Ok(event) => { - // Note: invocation_id filtering is no longer supported at event level - // since events are implicitly associated with invocations via the stream - if invocation_filter.is_some() { - // Ignore filter for now - would need to track at stream level - } - - // Apply timestamp filter if specified - if let Some(start_ts) = timestamp_filter { - let event_ts = match &event.event { - Some(proto::build_event::Event::BuildStarted(e)) => e.timestamp_millis, - Some(proto::build_event::Event::BuildFinished(e)) => e.timestamp_millis, - Some(proto::build_event::Event::TargetStarted(e)) => e.timestamp_millis, - Some(proto::build_event::Event::TargetCompleted(e)) => { - e.timestamp_millis - } - Some(proto::build_event::Event::ActionStarted(e)) => e.timestamp_millis, - Some(proto::build_event::Event::ActionCompleted(e)) => { - e.timestamp_millis - } - Some(proto::build_event::Event::BuildMetadata(_)) => 0, // Metadata events have no timestamp - Some(proto::build_event::Event::FileCreated(e)) => e.timestamp_millis, - None => return None, - }; - - if event_ts < start_ts { - return None; - } - } - - Some(Ok(event)) - } - Err(e) => { - warn!("Broadcast stream error: {}", e); - // Convert to gRPC status - Some(Err(tonic::Status::internal(format!("Stream error: {}", e)))) - } - } - }); - - Ok(tonic::Response::new(Box::pin(filtered_stream))) - } - - async fn publish_build_event( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let req = request.into_inner(); - - // Extract the event from the request - let event = req - .event - .ok_or_else(|| tonic::Status::invalid_argument("Missing event in request"))?; - - // Publish to broadcast channel - self.publish(event); - - // Return success response with empty name (will be set by server) - Ok(tonic::Response::new(proto::PublishBuildEventResponse { - name: String::new(), - })) - } - - type PublishBuildEventStreamStream = std::pin::Pin< - Box> + Send + 'static>, - >; - - async fn publish_build_event_stream( - &self, - _request: tonic::Request>, - ) -> Result, tonic::Status> { - // Not implemented in this service - this is a client-side method - Err(tonic::Status::unimplemented( - "Bidirectional streaming not implemented in this build events service" - )) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_service_creation() { - let service = BuildEventServiceImpl::new(100); - assert_eq!(service.subscriber_count(), 0); - } - - #[test] - fn test_publish_no_subscribers() { - let service = BuildEventServiceImpl::new(100); - let event = proto::BuildEvent { - invocation_id: String::new(), - event: None, - }; - // Should not panic - service.publish(event); - } - - #[tokio::test] - async fn test_subscriber_count() { - let service = BuildEventServiceImpl::new(100); - assert_eq!(service.subscriber_count(), 0); - - let _rx1 = service.event_sender.subscribe(); - assert_eq!(service.subscriber_count(), 1); - - let _rx2 = service.event_sender.subscribe(); - assert_eq!(service.subscriber_count(), 2); - } -} diff --git a/crates/build-events-proto/src/spongebob_subscriber_v2.rs b/crates/build-events-proto/src/spongebob_subscriber_v2.rs index 003253a5cf..6384b73d3e 100644 --- a/crates/build-events-proto/src/spongebob_subscriber_v2.rs +++ b/crates/build-events-proto/src/spongebob_subscriber_v2.rs @@ -1,26 +1,22 @@ -//! Subscriber that sends build events to Spongebob service using the workspace spongebob crate +//! Subscriber that sends build events to Spongebob service -#[cfg(feature = "spongebob-subscriber")] use async_trait::async_trait; -#[cfg(feature = "spongebob-subscriber")] use build_events::{BuildEvent, BuildEventSubscriber, SubscriberError}; -#[cfg(feature = "spongebob-subscriber")] use tracing::warn; /// Subscriber that publishes build events to Spongebob service /// -/// This subscriber uses the workspace `spongebob` crate to connect to the +/// This subscriber uses the `SpongeBob` client to connect to the /// Spongebob service and publish build events via bidirectional streaming. /// /// # Example -/// ```no_run -/// # #[cfg(feature = "spongebob-subscriber")] -/// use build_events_proto::SpongeBobSubscriberV2; +/// ```ignore +/// use build_events_proto::{SpongeBob, SpongeBobSubscriberV2}; /// /// #[tokio::main] /// async fn main() -> Result<(), Box> { /// // Create a spongebob client (opens stream and gets server-assigned ID) -/// let spongebob = spongebob::SpongeBob::new().await?; +/// let spongebob = SpongeBob::new().await?; /// /// // Create subscriber from the client /// let subscriber = SpongeBobSubscriberV2::from_client(spongebob); @@ -30,14 +26,12 @@ use tracing::warn; /// Ok(()) /// } /// ``` -#[cfg(feature = "spongebob-subscriber")] pub struct SpongeBobSubscriberV2 { - spongebob: std::sync::Arc, + spongebob: std::sync::Arc, /// Cache of file paths from BuildMetadata events, keyed by "{target_id}/stdout_path" etc. file_paths: std::sync::Arc>>, } -#[cfg(feature = "spongebob-subscriber")] impl SpongeBobSubscriberV2 { /// Create subscriber from an existing SpongeBob client /// @@ -45,16 +39,15 @@ impl SpongeBobSubscriberV2 { /// * `client` - An existing SpongeBob client with active streaming connection /// /// # Example - /// ```no_run - /// # #[cfg(feature = "spongebob-subscriber")] - /// # use build_events_proto::SpongeBobSubscriberV2; + /// ```ignore + /// # use build_events_proto::{SpongeBob, SpongeBobSubscriberV2}; /// # async fn example() -> Result<(), Box> { - /// let spongebob = spongebob::SpongeBob::new().await?; + /// let spongebob = SpongeBob::new().await?; /// let subscriber = SpongeBobSubscriberV2::from_client(spongebob); /// # Ok(()) /// # } /// ``` - pub fn from_client(client: spongebob::SpongeBob) -> Self { + pub fn from_client(client: crate::client::SpongeBob) -> Self { Self { spongebob: std::sync::Arc::new(client), file_paths: std::sync::Arc::new(tokio::sync::Mutex::new( @@ -64,7 +57,6 @@ impl SpongeBobSubscriberV2 { } } -#[cfg(feature = "spongebob-subscriber")] #[async_trait] impl BuildEventSubscriber for SpongeBobSubscriberV2 { async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { @@ -154,7 +146,7 @@ impl BuildEventSubscriber for SpongeBobSubscriberV2 { } } -#[cfg(all(test, feature = "spongebob-subscriber"))] +#[cfg(test)] mod tests { #[test] fn test_subscriber_creation() { diff --git a/crates/build-events-proto/src/subscriber.rs b/crates/build-events-proto/src/subscriber.rs deleted file mode 100644 index cb14831565..0000000000 --- a/crates/build-events-proto/src/subscriber.rs +++ /dev/null @@ -1,158 +0,0 @@ -//! gRPC streaming subscriber implementation -//! -//! Provides a [`GrpcStreamSubscriber`] that converts Rust events to protobuf -//! and makes them available as an async stream. - -use async_trait::async_trait; -use build_events::{BuildEvent, BuildEventSubscriber, SubscriberError}; -use tokio::sync::broadcast; -use tokio_stream::Stream; -use tokio_stream::wrappers::BroadcastStream; -use tracing::warn; - -use crate::proto; - -/// Subscriber that converts events to protobuf and provides them as a stream -/// -/// This subscriber implements [`BuildEventSubscriber`] and maintains an internal -/// broadcast channel for streaming events. It converts Rust events to protobuf -/// on reception and broadcasts them to all subscribers. -/// -/// # Example -/// ``` -/// use build_events_proto::GrpcStreamSubscriber; -/// -/// // Create subscriber -/// let subscriber = GrpcStreamSubscriber::new(100); -/// -/// // Can be used with BuildEventDispatcher -/// // dispatcher.add_subscriber(Box::new(subscriber)); -/// ``` -pub struct GrpcStreamSubscriber { - sender: broadcast::Sender, -} - -impl GrpcStreamSubscriber { - /// Create a new gRPC stream subscriber with the specified channel capacity - /// - /// The capacity determines how many events can be buffered before - /// slow subscribers start lagging. - /// - /// # Arguments - /// * `capacity` - Broadcast channel buffer capacity - /// - /// # Example - /// ``` - /// use build_events_proto::GrpcStreamSubscriber; - /// - /// let subscriber = GrpcStreamSubscriber::new(100); - /// ``` - pub fn new(capacity: usize) -> Self { - let (sender, _) = broadcast::channel(capacity); - Self { sender } - } - - /// Subscribe to the event stream - /// - /// Returns a stream that receives all events broadcasted by this subscriber. - /// Multiple subscriptions can be created, and each will receive all events. - /// - /// # Example - /// ``` - /// use build_events_proto::GrpcStreamSubscriber; - /// use tokio_stream::StreamExt; - /// - /// # #[tokio::main] - /// # async fn main() { - /// let subscriber = GrpcStreamSubscriber::new(100); - /// let mut stream = subscriber.subscribe_stream(); - /// - /// // Consume events - /// // while let Some(Ok(event)) = stream.next().await { - /// // // Process event - /// // } - /// # } - /// ``` - pub fn subscribe_stream(&self) -> impl Stream { - use tokio_stream::StreamExt; - BroadcastStream::new(self.sender.subscribe()).filter_map(|result| result.ok()) - } - - /// Get a broadcast sender for this subscriber - /// - /// Useful for advanced use cases where you need direct access to the sender. - pub fn sender(&self) -> broadcast::Sender { - self.sender.clone() - } -} - -#[async_trait] -impl BuildEventSubscriber for GrpcStreamSubscriber { - async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { - // Broadcast to all subscribers - // If there are no subscribers, this is a no-op - if self.sender.receiver_count() == 0 { - return Ok(()); - } - - // Send to broadcast channel (will drop if no receivers) - if let Err(e) = self.sender.send(event.clone()) { - // If channel has no receivers, this is expected - warn!("GrpcStreamSubscriber: no active receivers ({})", e); - } - - Ok(()) - } - - fn name(&self) -> &str { - "GrpcStreamSubscriber" - } - - async fn on_close(&self) -> Result<(), SubscriberError> { - // Channel will automatically close when sender is dropped - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use build_events::events::{BuildStarted, current_millis}; - use tokio_stream::StreamExt; - - #[tokio::test] - async fn test_subscriber_stream() { - use build_events::events::build_event; - - let subscriber = GrpcStreamSubscriber::new(10); - let mut stream = subscriber.subscribe_stream(); - - let event = BuildEvent { - invocation_id: "test-123".to_string(), - event: Some(build_event::Event::BuildStarted(BuildStarted { - invocation_id: "test-123".to_string(), - command: "test".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - })), - }; - - // Send event - subscriber.on_event(&event).await.unwrap(); - - // Receive from stream - let received = tokio::time::timeout(std::time::Duration::from_millis(100), stream.next()) - .await - .unwrap(); - - assert!(received.is_some()); - let proto_event = received.unwrap(); - assert!(proto_event.event.is_some()); - } - - #[tokio::test] - async fn test_subscriber_name() { - let subscriber = GrpcStreamSubscriber::new(10); - assert_eq!(subscriber.name(), "GrpcStreamSubscriber"); - } -} diff --git a/crates/minimal/Cargo.toml b/crates/minimal/Cargo.toml index adbe829488..36ecad8e0a 100644 --- a/crates/minimal/Cargo.toml +++ b/crates/minimal/Cargo.toml @@ -6,9 +6,8 @@ publish.workspace = true [dependencies] "build-sandbox".workspace = true -spongebob = { path = "../spongebob" } build-events.workspace = true -build-events-proto = { path = "../build-events-proto", features = ["spongebob-subscriber"] } +build-events-proto = { path = "../build-events-proto" } graph.workspace = true cache.workspace = true common.workspace = true diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index 00f023cb3d..f2d92c40ec 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -21,7 +21,6 @@ pub async fn cmd_build(args: BuildArgs, globals: &GlobalArgs) -> Result<(), Erro let graph = args.packages.graph(globals)?; let cache = globals.cache().map_err(anyhow::Error::from)?; - cmd_build_impl(&graph, globals, cache.clone(), globals.num_parallel_builds).await?; cmd_build_impl(&graph, globals, cache, globals.num_parallel_builds).await?; Ok(()) @@ -215,7 +214,7 @@ pub async fn cmd_build_impl( // Display build summary with spongebob URL let spongebob_url = format!( - "https://dash.minimal.dev/invocations/{}", + "https://dash.minimal.farm/invocations/{}", build_events::invocation_id() ); display_build_summary(graph, &cache, globals, &run, Some(&spongebob_url)); diff --git a/crates/minimal/src/main.rs b/crates/minimal/src/main.rs index 9a52d02d24..625c364561 100644 --- a/crates/minimal/src/main.rs +++ b/crates/minimal/src/main.rs @@ -363,7 +363,7 @@ async fn main() -> Result<()> { cli.global_args.path_config().ensure_directories()?; // Initialize global event bus and SpongeBob integration - let spongebob = spongebob::SpongeBob::new() + let spongebob = build_events_proto::SpongeBob::new() .await .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; let invocation_id = spongebob.invocation_id().to_string(); diff --git a/crates/spongebob/Cargo.toml b/crates/spongebob/Cargo.toml deleted file mode 100644 index 4c55232238..0000000000 --- a/crates/spongebob/Cargo.toml +++ /dev/null @@ -1,14 +0,0 @@ -[package] -name = "spongebob" -version = "0.1.0" -edition.workspace = true -publish.workspace = true - -[dependencies] -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014214004-4ceda9b7e17f.1", registry = "buf" } -minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251014214004-4ceda9b7e17f.1", registry = "buf" } -thiserror.workspace = true -tokio = { version = "1", features = ["sync"] } -tokio-stream = "0.1" -tonic = { version = "0.12.3", features = ["tls", "tls-native-roots"] } -tracing.workspace = true From f3b58aebbf45b8e2fbf5ae3f24c0431ac5207000 Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 14 Oct 2025 11:35:08 -0700 Subject: [PATCH 20/39] crates/cache: fix CI flake caused by wall clock --- crates/cache/src/lib.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index 17d2aa1416..93cf9e24a0 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -490,16 +490,16 @@ mod tests { let key2 = SpecHash(blake3::hash("key2".as_bytes())); let w = cache.write_dir(&key1).unwrap(); + let m = EntryMeta { + spec_name: "spec".to_string(), + fetched: false, + ..Default::default() + }; w.open_write("file_name") .unwrap() .write_all("old data".as_bytes()) .unwrap(); - w.finalize(EntryMeta { - spec_name: "spec".to_string(), - fetched: false, - ..Default::default() - }) - .unwrap(); + w.finalize(m.clone()).unwrap(); // Write again and change the data in the file let w = cache.write_dir(&key2).unwrap(); @@ -510,6 +510,7 @@ mod tests { w.finalize(EntryMeta { spec_name: "spec".to_string(), fetched: true, + epoch_millis: m.epoch_millis + 1000, ..Default::default() }) .unwrap(); From 90b36f3ca349325da7b15593456d72a383f4583a Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 14 Oct 2025 12:42:34 -0700 Subject: [PATCH 21/39] crates/cache: fix clippy --- crates/cache/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index 93cf9e24a0..734359904a 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -511,7 +511,6 @@ mod tests { spec_name: "spec".to_string(), fetched: true, epoch_millis: m.epoch_millis + 1000, - ..Default::default() }) .unwrap(); From 4c1b2f646c507f9361260f45401f3d34f0a6477e Mon Sep 17 00:00:00 2001 From: Jake K Date: Tue, 14 Oct 2025 13:06:43 -0700 Subject: [PATCH 22/39] addded btop 1.4.5 package def + build script --- packages/btop/build.ncl | 27 +++++++++++++++++++++++++++ packages/btop/build.sh | 7 +++++++ 2 files changed, 34 insertions(+) create mode 100644 packages/btop/build.ncl create mode 100755 packages/btop/build.sh diff --git a/packages/btop/build.ncl b/packages/btop/build.ncl new file mode 100644 index 0000000000..04648c03b2 --- /dev/null +++ b/packages/btop/build.ncl @@ -0,0 +1,27 @@ +let { BuildSpec, Local, OutputBin, OutputData, Source, .. } = import "minimal.ncl" in +let base = import "../base/build.ncl" in +let make = import "../make/build.ncl" in +let toolchain = import "../toolchain/build.ncl" in +let ncurses = import "../ncurses/build.ncl" in + +let version = "1.4.5" in +{ + name = "btop", + inputs = [ + { file = "build.sh" } | Local, + { + url = "https://github.com/aristocratos/btop/archive/refs/tags/v%{version}.tar.gz", + sha256 = "0ffe03d3e26a3e9bbfd5375adf34934137757994f297d6b699a46edd43c3fc02", + extract = true, + } | Source, + base, + make, + toolchain, + ], + runtime_deps = [ncurses], + cmd = "./build.sh", + outputs = { + btop = { glob = "usr/bin/btop" } | OutputBin, + data = { glob = "usr/share/**/*" } | OutputData, + }, +} | BuildSpec diff --git a/packages/btop/build.sh b/packages/btop/build.sh new file mode 100755 index 0000000000..bbe69499f2 --- /dev/null +++ b/packages/btop/build.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -e +cd btop-1.4.5 + +# Disable GPU paths at compile time +make -j"$(nproc)" GPU_SUPPORT=false +make DESTDIR="${OUTPUT_DIR}" PREFIX="/usr" install From 828cdb9d9b21027aa164782116baa7136cfbd686 Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 14 Oct 2025 13:20:01 -0700 Subject: [PATCH 23/39] crates/cache: shard metadata based on the first byte --- crates/cache/src/entry_meta.rs | 64 +++++++++++++++++-------- crates/cache/src/lib.rs | 6 +-- crates/minimal/src/cmd_patched_build.rs | 2 +- 3 files changed, 47 insertions(+), 25 deletions(-) diff --git a/crates/cache/src/entry_meta.rs b/crates/cache/src/entry_meta.rs index 03175d0791..b1c7d2b4fd 100644 --- a/crates/cache/src/entry_meta.rs +++ b/crates/cache/src/entry_meta.rs @@ -32,7 +32,14 @@ impl EntryMeta { /// Writes this metadata to the cache filesystem for the given hash. pub fn write(&self, fs: &FS, hash: &SpecHash) -> Result<(), CacheErr> { let hash_hex = hash.0.to_hex(); - let f = fs.open_write(format!("meta/{}.json", hash_hex))?; + let dir_hex = hash.as_bytes()[0]; + fs.mkdir(format!("meta/{:02x}", dir_hex)).ok(); //ignore error + + let f = fs.open_write(format!( + "meta/{:02x}/{}.json", + dir_hex, + &hash_hex.as_str()[2..] + ))?; serde_json::to_writer(f, self).map_err(std::io::Error::from)?; Ok(()) } @@ -40,42 +47,57 @@ impl EntryMeta { /// Reads metadata from the cache filesystem for the given hash. pub fn read(fs: &FS, hash: &SpecHash) -> Result { let hash_hex = hash.0.to_hex(); - let f = fs.open_read(format!("meta/{}.json", hash_hex))?; + let dir_hex = hash.as_bytes()[0]; + + let f = fs.open_read(format!( + "meta/{:02x}/{}.json", + dir_hex, + &hash_hex.as_str()[2..] + ))?; serde_json::from_reader(f).map_err(|e| CacheErr::IO(std::io::Error::from(e))) } /// Reads metadata from a file handle. - pub fn read_from(reader: R) -> Result { + pub(crate) fn read_from(reader: R) -> Result { serde_json::from_reader(reader).map_err(|e| CacheErr::IO(std::io::Error::from(e))) } /// Lists all metadata entries in the cache that match a given spec name. /// /// Returns a vector of tuples containing (hash_hex, epoch_millis) sorted by recency. - pub fn find_by_name( + pub fn find_build_by_name( fs: &FS, name: &str, ) -> Result, CacheErr> { let mut candidates = Vec::new(); - for e in fs.read_dir("meta")? { - let f = fs.open_read(e.path()?)?; - let entry: EntryMeta = Self::read_from(f)?; - if entry.spec_name == name { - let p = e.path()?; - let hash_hex = p - .file_name() - .and_then(|n| n.to_str()) - .and_then(|s| s.strip_suffix(".json")) - .ok_or_else(|| { - CacheErr::IO(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "invalid metadata filename", - )) - })? - .to_ascii_lowercase(); + for dir in fs.read_dir("meta")? { + let leading_hex = dir + .path()? + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_owned(); + + for e in fs.read_dir(dir.path()?)? { + let f = fs.open_read(e.path()?)?; + let entry: EntryMeta = Self::read_from(f)?; + if entry.spec_name == name { + let p = e.path()?; + let hash_hex = leading_hex.to_owned() + + p.file_name() + .and_then(|n| n.to_str()) + .and_then(|s| s.strip_suffix(".json")) + .ok_or_else(|| { + CacheErr::IO(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "invalid metadata filename", + )) + })?; - candidates.push((hash_hex, entry.epoch_millis)); + candidates.push((hash_hex, entry.epoch_millis)); + } } } diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index 734359904a..53c4fecbc4 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -310,10 +310,10 @@ impl Cache { /// Returns the cache entry that was most recently generated by a spec of the given name. /// /// NOTE: This method breaks the caching model and should not be used unless you know what you are doing!! - pub fn unsafe_get_by_name(&self, name: &str) -> Result, CacheErr> { + pub fn unsafe_get_build_by_name(&self, name: &str) -> Result, CacheErr> { let candidates = { let inner = self.inner(); - EntryMeta::find_by_name(&inner.fs, name)? + EntryMeta::find_build_by_name(&inner.fs, name)? }; if let Some((spec_hash_hex, _)) = candidates.first() { @@ -515,7 +515,7 @@ mod tests { .unwrap(); // Make sure the returned dir was the latest write with that name - let dir = cache.unsafe_get_by_name("spec").unwrap(); + let dir = cache.unsafe_get_build_by_name("spec").unwrap(); let r = dir.open_read("file_name").unwrap(); assert_eq!("new data", std::io::read_to_string(r).unwrap()); assert_eq!(dir.hash, key2); diff --git a/crates/minimal/src/cmd_patched_build.rs b/crates/minimal/src/cmd_patched_build.rs index 39d2ebae36..7f6e346e22 100644 --- a/crates/minimal/src/cmd_patched_build.rs +++ b/crates/minimal/src/cmd_patched_build.rs @@ -33,7 +33,7 @@ pub async fn cmd_patched_build(args: PatchedBuildArgs, globals: &GlobalArgs) -> .collect(); for bsr in build_deps.iter() { let build = graph.get(bsr).unwrap(); - let cache_dir = cache.unsafe_get_by_name(&build.name).unwrap(); + let cache_dir = cache.unsafe_get_build_by_name(&build.name).unwrap(); dependencies.insert(cache_dir.path().to_path_buf()); } From b06f9eb9334d0214035094d1d585a8de2de8497e Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 14 Oct 2025 13:29:47 -0700 Subject: [PATCH 24/39] minimal: surface better error when sources fail to fetch --- crates/minimal/src/remote_storage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/minimal/src/remote_storage.rs b/crates/minimal/src/remote_storage.rs index 896a2458df..859f1229a5 100644 --- a/crates/minimal/src/remote_storage.rs +++ b/crates/minimal/src/remote_storage.rs @@ -90,7 +90,7 @@ impl RemoteStorage { let mut hasher = Sha256::new(); { let mut f = fs::File::create(&cached_path)?; - let mut res = reqwest::get(url).await?; + let mut res = reqwest::get(url).await?.error_for_status()?; let mut w = Tee::new(&mut f, &mut hasher); while let Some(chunk) = res.chunk().await? { From beae0b0f8eae0004cac28ebc91f1639ae77e0a8e Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 14 Oct 2025 14:24:12 -0700 Subject: [PATCH 25/39] crates/cache: start wiring metadata for storing subsets in the cache --- Cargo.lock | 1 + crates/cache/Cargo.toml | 1 + crates/cache/src/entry_meta.rs | 70 ++++++++++++++++++------- crates/cache/src/lib.rs | 22 ++++---- crates/cache/src/remote.rs | 5 +- crates/common/src/lib.rs | 2 + crates/common/src/subsets.rs | 16 ++++++ crates/minimal/src/cmd_patched_build.rs | 4 +- crates/minimal/src/run.rs | 4 +- 9 files changed, 90 insertions(+), 35 deletions(-) create mode 100644 crates/common/src/subsets.rs diff --git a/Cargo.lock b/Cargo.lock index 450baeae49..c0f3608740 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -434,6 +434,7 @@ dependencies = [ "sha2", "tempfile", "tokio", + "tracing", "url", ] diff --git a/crates/cache/Cargo.toml b/crates/cache/Cargo.toml index 93e20463b4..92019acb09 100644 --- a/crates/cache/Cargo.toml +++ b/crates/cache/Cargo.toml @@ -10,6 +10,7 @@ publish.workspace = true indoc.workspace = true [dependencies] +tracing.workspace = true blake3.workspace = true hex.workspace = true sha2.workspace = true diff --git a/crates/cache/src/entry_meta.rs b/crates/cache/src/entry_meta.rs index b1c7d2b4fd..ae60d492ad 100644 --- a/crates/cache/src/entry_meta.rs +++ b/crates/cache/src/entry_meta.rs @@ -1,16 +1,33 @@ //! Metadata for cache entries. -use common::SpecHash; +use common::{SpecHash, SubsetSpec}; use std::io::Read; use std::time::{SystemTime, UNIX_EPOCH}; +use tracing::warn; use crate::CacheErr; use crate::fs::{DirEntry, FileSystem}; +/// Metadata specific to the object being cached. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +pub enum MetaInner { + Spec(String), // name + Subset(SubsetSpec), // (build-spec, output-names), both sorted +} + +impl MetaInner { + pub fn spec_name(&self) -> Option<&String> { + match self { + MetaInner::Spec(sn) => Some(sn), + _ => None, + } + } +} + /// Metadata associated with a cache entry. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] pub struct EntryMeta { - pub spec_name: String, + pub inner: MetaInner, pub fetched: bool, pub epoch_millis: u128, } @@ -18,7 +35,7 @@ pub struct EntryMeta { impl Default for EntryMeta { fn default() -> Self { EntryMeta { - spec_name: "".to_string(), + inner: MetaInner::Spec("".to_string()), fetched: false, epoch_millis: SystemTime::now() .duration_since(UNIX_EPOCH) @@ -82,21 +99,36 @@ impl EntryMeta { for e in fs.read_dir(dir.path()?)? { let f = fs.open_read(e.path()?)?; - let entry: EntryMeta = Self::read_from(f)?; - if entry.spec_name == name { - let p = e.path()?; - let hash_hex = leading_hex.to_owned() - + p.file_name() - .and_then(|n| n.to_str()) - .and_then(|s| s.strip_suffix(".json")) - .ok_or_else(|| { - CacheErr::IO(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "invalid metadata filename", - )) - })?; - - candidates.push((hash_hex, entry.epoch_millis)); + let entry = match Self::read_from(f) { + Ok(entry) => entry, + Err(e) => { + warn!( + "Skipping cache metadata entry that failed to parse: {:?}", + e + ); + continue; + } + }; + + match entry.inner { + MetaInner::Subset(_) => {} + MetaInner::Spec(spec_name) => { + if spec_name == name { + let p = e.path()?; + let hash_hex = leading_hex.to_owned() + + p.file_name() + .and_then(|n| n.to_str()) + .and_then(|s| s.strip_suffix(".json")) + .ok_or_else(|| { + CacheErr::IO(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "invalid metadata filename", + )) + })?; + + candidates.push((hash_hex, entry.epoch_millis)); + } + } } } } diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index 53c4fecbc4..7d948d2942 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -15,7 +15,7 @@ mod remote; pub use remote::{Error as RemoteError, RemoteCache}; pub(crate) mod entry_meta; -pub use entry_meta::EntryMeta; +pub use entry_meta::{EntryMeta, MetaInner}; #[allow(dead_code)] mod remote_index; @@ -364,6 +364,8 @@ impl<'a, B: common::fetchers::FetchBackend> graph::BinProvider for RemoteBinProv #[cfg(test)] mod tests { + use crate::entry_meta::MetaInner; + use super::*; use tempfile::TempDir; @@ -381,7 +383,7 @@ mod tests { .write_all("uwu".as_bytes()) .unwrap(); w.finalize(EntryMeta { - spec_name: "some spec huh".to_string(), + inner: MetaInner::Spec("some spec huh".to_string()), fetched: false, ..Default::default() }) @@ -389,7 +391,7 @@ mod tests { let meta = EntryMeta::read(&cache.inner().fs, &test_key).unwrap(); assert!(!meta.fetched); - assert_eq!(meta.spec_name, "some spec huh"); + assert_eq!(meta.inner.spec_name(), Some(&"some spec huh".to_string())); let r = cache .read_dir(&test_key) @@ -413,7 +415,7 @@ mod tests { .write_all("uwu".as_bytes()) .unwrap(); w.finalize(EntryMeta { - spec_name: "old name".to_string(), + inner: MetaInner::Spec("old name".to_string()), fetched: false, ..Default::default() }) @@ -434,7 +436,7 @@ mod tests { .unwrap(); assert_eq!("uwu", std::io::read_to_string(r).unwrap()); let meta = EntryMeta::read(&cache.inner().fs, &test_key).unwrap(); - assert_eq!(meta.spec_name, "old name"); + assert_eq!(meta.inner.spec_name(), Some(&"old name".to_string())); } #[test] @@ -451,7 +453,7 @@ mod tests { .write_all("bad data".as_bytes()) .unwrap(); w.finalize(EntryMeta { - spec_name: "old".to_string(), + inner: MetaInner::Spec("old".to_string()), fetched: false, ..Default::default() }) @@ -464,7 +466,7 @@ mod tests { .write_all("good data".as_bytes()) .unwrap(); w.finalize(EntryMeta { - spec_name: "new".to_string(), + inner: MetaInner::Spec("new".to_string()), fetched: false, ..Default::default() }) @@ -477,7 +479,7 @@ mod tests { .unwrap(); assert_eq!("good data", std::io::read_to_string(r).unwrap()); let meta = EntryMeta::read(&cache.inner().fs, &test_key).unwrap(); - assert_eq!(meta.spec_name, "new"); + assert_eq!(meta.inner.spec_name(), Some(&"new".to_string())); } #[test] @@ -491,7 +493,7 @@ mod tests { let w = cache.write_dir(&key1).unwrap(); let m = EntryMeta { - spec_name: "spec".to_string(), + inner: MetaInner::Spec("spec".to_string()), fetched: false, ..Default::default() }; @@ -508,7 +510,7 @@ mod tests { .write_all("new data".as_bytes()) .unwrap(); w.finalize(EntryMeta { - spec_name: "spec".to_string(), + inner: MetaInner::Spec("spec".to_string()), fetched: true, epoch_millis: m.epoch_millis + 1000, }) diff --git a/crates/cache/src/remote.rs b/crates/cache/src/remote.rs index 6eaddc7406..ae1a5d597a 100644 --- a/crates/cache/src/remote.rs +++ b/crates/cache/src/remote.rs @@ -198,9 +198,10 @@ impl RemoteCache { ) .map_err(Error::ArchiveError)?; + use crate::{EntryMeta, MetaInner}; cache_hnd - .finalize(crate::EntryMeta { - spec_name: spec_name.to_string(), + .finalize(EntryMeta { + inner: MetaInner::Spec(spec_name.to_string()), fetched: true, ..Default::default() }) diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 89ddf10cfc..1ee834c5fe 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -8,10 +8,12 @@ pub mod archive; mod spec_hash; pub use spec_hash::SpecHash; +mod subsets; use std::{ io::Write, path::{Path, PathBuf}, }; +pub use subsets::SubsetSpec; use tracing::warn; /// Implements [Write], mirroring all writes to two underlying writers. diff --git a/crates/common/src/subsets.rs b/crates/common/src/subsets.rs new file mode 100644 index 0000000000..2591aa40b0 --- /dev/null +++ b/crates/common/src/subsets.rs @@ -0,0 +1,16 @@ +use crate::SpecHash; + +/// Description of a subset. +/// +/// A subset is a subset of the build artifacts from other builds, bundled together. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +pub struct SubsetSpec { + inner: Vec<(SpecHash, Vec)>, +} + +impl SubsetSpec { + /// Returns the SpecHashes of builds this subset is composed from. + pub fn depends_on(&self) -> impl Iterator { + self.inner.iter().map(|e| &e.0) + } +} diff --git a/crates/minimal/src/cmd_patched_build.rs b/crates/minimal/src/cmd_patched_build.rs index 7f6e346e22..087a368c8c 100644 --- a/crates/minimal/src/cmd_patched_build.rs +++ b/crates/minimal/src/cmd_patched_build.rs @@ -2,7 +2,7 @@ use crate::GlobalArgs; use crate::run::Materialized; use crate::{Error, remote_storage::RemoteStorage}; use build_sandbox::{BuildConfig, Input as SandboxInput, config::BuildScript, run_build}; -use cache::EntryMeta; +use cache::{EntryMeta, MetaInner}; use graph::{BuildOutput, BuildSpecInput, Transitives}; use std::collections::HashSet; use tracing::info; @@ -111,7 +111,7 @@ pub async fn cmd_patched_build(args: PatchedBuildArgs, globals: &GlobalArgs) -> out_dir .finalize(EntryMeta { - spec_name: build.name.clone(), + inner: MetaInner::Spec(build.name.clone()), ..Default::default() }) .unwrap(); diff --git a/crates/minimal/src/run.rs b/crates/minimal/src/run.rs index cf201fef84..8a00fdba05 100644 --- a/crates/minimal/src/run.rs +++ b/crates/minimal/src/run.rs @@ -1,6 +1,6 @@ use anyhow::{Context, Result, bail}; use build_sandbox::{BuildConfig, Input as SandboxInput, config::BuildScript, run_build}; -use cache::{Cache, LocalDir, PendingDir, RemoteCache}; +use cache::{Cache, LocalDir, MetaInner, PendingDir, RemoteCache}; use common::SpecHash; use google_cloud_storage::client::Storage as GcsStorage; use graph::dep_graph::SourceFetch; @@ -619,7 +619,7 @@ impl<'a> Run<'a> { .for_each(|(cache_hnd, name)| { cache_hnd .finalize(cache::EntryMeta { - spec_name: name, + inner: MetaInner::Spec(name), fetched: false, ..Default::default() }) From 8f38d939aea1932080ba955d7d0c6c8681e1e6aa Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 14 Oct 2025 16:50:34 -0700 Subject: [PATCH 26/39] packages: fix runtime_deps of pciutils --- packages/pciutils/build.ncl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pciutils/build.ncl b/packages/pciutils/build.ncl index 7ef496bbf3..ef871a33a8 100644 --- a/packages/pciutils/build.ncl +++ b/packages/pciutils/build.ncl @@ -1,9 +1,9 @@ let { BuildSpec, Local, OutputBin, OutputData, OutputLib, Source, .. } = import "minimal.ncl" in let base = import "../base/build.ncl" in let sed = import "../sed/build.ncl" in -let gcc = import "../gcc/build.ncl" in let make = import "../make/build.ncl" in let toolchain = import "../toolchain/build.ncl" in +let glibc = import "../glibc/build.ncl" in let zlib = import "../zlib/build.ncl" in { name = "pciutils", @@ -19,8 +19,8 @@ let zlib = import "../zlib/build.ncl" in toolchain, ], runtime_deps = [ + glibc, zlib, - gcc, # TODO: Just needs libresolv ], cmd = "./build.sh", outputs = { From f3126b00e518f6499e1d79e540a9bbfc2c0562f8 Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 14 Oct 2025 17:20:17 -0700 Subject: [PATCH 27/39] minimal: wiring for storing subsets in the cache --- Cargo.lock | 2 +- crates/cache/src/remote.rs | 7 +- crates/common/Cargo.toml | 2 + crates/common/src/lib.rs | 59 +++++++++ crates/common/src/subsets.rs | 35 +++++- crates/graph/minimal-ncl/minimal.ncl | 2 - crates/graph/src/dep_graph.rs | 50 +++++--- crates/graph/src/spec_hasher.rs | 98 +++++++++++++-- crates/minimal/Cargo.toml | 1 - crates/minimal/src/cmd_build.rs | 2 +- crates/minimal/src/cmd_check/outputs.rs | 54 ++------- crates/minimal/src/main.rs | 9 ++ crates/minimal/src/run.rs | 154 ++++++++++++++++++++++-- packages/icu/build.ncl | 4 +- 14 files changed, 380 insertions(+), 99 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0f3608740..bf2da861ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -561,6 +561,7 @@ dependencies = [ "bytes", "flate2", "futures", + "globset", "google-cloud-gax", "google-cloud-storage", "indoc", @@ -2192,7 +2193,6 @@ dependencies = [ "flate2", "futures", "generational-arena", - "globset", "google-cloud-gax", "google-cloud-storage", "graph", diff --git a/crates/cache/src/remote.rs b/crates/cache/src/remote.rs index ae1a5d597a..a62e54f16f 100644 --- a/crates/cache/src/remote.rs +++ b/crates/cache/src/remote.rs @@ -1,4 +1,4 @@ -use crate::{Cache, LocalDir, remote_index::RemoteIndex}; +use crate::{Cache, EntryMeta, LocalDir, MetaInner, remote_index::RemoteIndex}; use common::{SpecHash, archive}; use std::io::{Seek, Write}; @@ -168,7 +168,7 @@ impl RemoteCache { pub async fn materialize( &self, spec_hash: &SpecHash, - spec_name: &str, + inner: MetaInner, cache: &Cache, ) -> Result<(), Error<::Error>> { let sha256: [u8; 32] = self.index.sha256(spec_hash).ok_or(Error::NotFound)?; @@ -198,10 +198,9 @@ impl RemoteCache { ) .map_err(Error::ArchiveError)?; - use crate::{EntryMeta, MetaInner}; cache_hnd .finalize(EntryMeta { - inner: MetaInner::Spec(spec_name.to_string()), + inner, fetched: true, ..Default::default() }) diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 736c4023d3..93ab81b33d 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -18,6 +18,8 @@ blake3.workspace = true smallvec.workspace = true tracing.workspace = true +globset.workspace = true + sha2.workspace = true tar.workspace = true flate2.workspace = true diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 1ee834c5fe..9a6f9314db 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -104,3 +104,62 @@ pub fn hardlink_dir_contents(src: &Path, dst: &Path) -> Result<(), HardlinkError Ok(()) } + +#[derive(Debug, thiserror::Error)] +pub enum GlobError { + #[error("IO error: {0}")] + IO(std::io::Error), + #[error("Glob error: {0}")] + Glob(globset::Error), +} + +impl From for GlobError { + fn from(e: globset::Error) -> Self { + GlobError::Glob(e) + } +} + +impl From for GlobError { + fn from(e: std::io::Error) -> Self { + GlobError::IO(e) + } +} + +/// Enumerates files which match the given glob within the given directory. +pub fn match_files_for_glob(dir: &Path, glob: &str) -> Result, GlobError> { + let mut results = Vec::new(); + + let matcher = globset::GlobBuilder::new(glob) + .literal_separator(true) + .empty_alternates(true) + .build()? + .compile_matcher(); + + // Walk the filesystem starting from root_dir + fn walk_dir( + dir: &Path, + root_dir: &Path, + matcher: &globset::GlobMatcher, + results: &mut Vec, + ) -> Result<(), GlobError> { + let entries = std::fs::read_dir(dir)?; + + for entry in entries { + let entry = entry?; + + let path = entry.path(); + let subset = path.strip_prefix(root_dir).unwrap(); + if path.is_file() && matcher.is_match(subset) { + results.push(path); + } else if path.is_dir() { + // Recursively walk subdirectories + walk_dir(&path, root_dir, matcher, results)?; + } + } + + Ok(()) + } + + walk_dir(dir, dir, &matcher, &mut results)?; + Ok(results) +} diff --git a/crates/common/src/subsets.rs b/crates/common/src/subsets.rs index 2591aa40b0..457c75c378 100644 --- a/crates/common/src/subsets.rs +++ b/crates/common/src/subsets.rs @@ -1,10 +1,11 @@ use crate::SpecHash; -/// Description of a subset. +/// Exact summary of a subset. /// /// A subset is a subset of the build artifacts from other builds, bundled together. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] pub struct SubsetSpec { + /// inner MUST be sorted: by spec hash, then by output name. inner: Vec<(SpecHash, Vec)>, } @@ -13,4 +14,36 @@ impl SubsetSpec { pub fn depends_on(&self) -> impl Iterator { self.inner.iter().map(|e| &e.0) } + + /// Constructs a new subset spec, for a subset which collects a subset of the + /// outputs from a single build-spec. + pub fn new_single(spec_hash: &SpecHash, mut outputs: Vec) -> Self { + outputs.sort(); + outputs.dedup(); + + Self { + inner: vec![(spec_hash.clone(), outputs)], + } + } + + pub fn iter_components(&self) -> impl Iterator)> { + self.inner.iter() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn outputs_sorted() { + let ss = SubsetSpec::new_single( + &SpecHash::from_hex("48d0229ccc7973402752c4a62a5d09875c5db3c426d316555207106200000000") + .unwrap(), + vec!["zAbc".to_string(), "abc".to_string()], + ); + + let outputs = ss.iter_components().next().unwrap().1.clone(); + assert!(outputs.is_sorted()); + } } diff --git a/crates/graph/minimal-ncl/minimal.ncl b/crates/graph/minimal-ncl/minimal.ncl index a19819f4ff..fc0f7cf059 100644 --- a/crates/graph/minimal-ncl/minimal.ncl +++ b/crates/graph/minimal-ncl/minimal.ncl @@ -121,8 +121,6 @@ in | BuildSpec, outputs | Array String, - strict - | Bool | optional = false, }, subsetOf | BuildSpec -> Array String -> Subset | doc "Constructs a subset of the outputs of a build, for use as an input or runtime_dep." = diff --git a/crates/graph/src/dep_graph.rs b/crates/graph/src/dep_graph.rs index d51138e828..f0e1b5013e 100644 --- a/crates/graph/src/dep_graph.rs +++ b/crates/graph/src/dep_graph.rs @@ -4,7 +4,7 @@ #![allow(clippy::single_match)] use common::{ - Target, + SubsetSpec, Target, target::{Arch, OS}, }; use nickel_lang_core::eval::Closure; @@ -19,8 +19,8 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::{Arc, RwLock}; -use crate::spec_schema::*; use crate::{Error, SpecError, SpecHash, SpecHasher, SpecReader}; +use crate::{spec_hasher::SubsetHasher, spec_schema::*}; use serde::Deserialize; /// A map with ordered iteration semantics - we need this for stable spec hashes. @@ -50,7 +50,20 @@ pub struct SourceInput { pub struct SubsetInput { pub from: BuildSpecRef, pub outputs: Vec, - pub strict: bool, +} + +impl SubsetInput { + /// Constructs a new [SubsetInput], except using the given build instead. + pub fn override_build(&self, bsr: BuildSpecRef) -> Self { + Self { + from: bsr, + ..self.clone() + } + } + + pub fn as_spec(&self, graph: &DepGraph) -> SubsetSpec { + SubsetSpec::new_single(&graph.spec_hash(&self.from), self.outputs.clone()) + } } /// A description of where a build was declared. @@ -97,6 +110,16 @@ pub enum BuildOutput { Binary { glob: String }, } +impl BuildOutput { + pub fn glob(&self) -> &String { + match self { + BuildOutput::Binary { glob } => glob, + BuildOutput::Data { glob } => glob, + BuildOutput::Library { glob } => glob, + } + } +} + /// A runtime dependency declared on a build-spec. /// /// Each entry in a build-spec's `runtime_deps` array corresponds to one [RuntimeDep]. @@ -248,6 +271,11 @@ impl DepGraph { hash } + /// Returns the specification hash of a subset. + pub fn subset_hash(&self, subset: &SubsetInput) -> SpecHash { + SubsetHasher::hash_single(self, &subset.from, subset.outputs.clone()) + } + /// Returns an iterator over all build-spec references. pub fn all(&self) -> impl Iterator + use<'_> { self.builds.iter().map(|e| BuildSpecRef(e.0)) @@ -698,7 +726,6 @@ impl GraphBuilder { ) -> Result { let mut from: Option = None; let mut outputs: Option> = None; - let mut strict: bool = false; match rt.term.as_ref() { Term::Record(r) | Term::RecRecord(r, _, _, _) => { r.fields @@ -737,13 +764,6 @@ impl GraphBuilder { } Ok(()) } - "strict" => { - if let Some(field) = field.value.as_ref() { - strict = bool::deserialize(eval_if_closure(field, program)?) - .unwrap(); - } - Ok(()) - } _ => Ok(()), // TODO: Should we error if we see an unknown field? } })?; @@ -786,11 +806,7 @@ impl GraphBuilder { } } - Ok(SubsetInput { - from, - strict, - outputs, - }) + Ok(SubsetInput { from, outputs }) } fn read_input_source( @@ -1415,7 +1431,6 @@ mod tests { dp.get(&dp.by_name("top").next().unwrap()).unwrap().inputs[0].clone(), BuildSpecInput::Subset(SubsetInput { from, - strict: false, outputs, }) if from == dp.by_name("other").next().unwrap() && outputs == vec!["abc", "uwu"], )); @@ -1665,7 +1680,6 @@ mod tests { dp.get(&dp.by_name("top build").next().unwrap()).unwrap().runtime_deps[1].clone(), RuntimeDep::Subset(SubsetInput { from, - strict: false, outputs, }) if from == dp.by_name("runtime dep").next().unwrap() && outputs == vec!["some_data"], )); diff --git a/crates/graph/src/spec_hasher.rs b/crates/graph/src/spec_hasher.rs index 3712c9cdc8..bd4f1616e9 100644 --- a/crates/graph/src/spec_hasher.rs +++ b/crates/graph/src/spec_hasher.rs @@ -3,7 +3,7 @@ use crate::{ dep_graph::SourceFetch, }; use blake3::Hasher; -use common::{Target, target}; +use common::{SubsetSpec, Target, target}; use smallvec::SmallVec; use std::collections::HashMap; use std::io::Write; @@ -14,7 +14,7 @@ type Edges = SmallVec<[Edge; 12]>; struct SpecIndex(usize); #[derive(Debug, Clone, PartialEq, Eq)] -struct SubsetInfo(Vec, bool); +struct SubsetInfo(Vec); #[derive(Debug, Clone, PartialEq, Eq)] enum Edge { @@ -56,25 +56,25 @@ impl<'a> SpecHasher<'a> { BuildInput(input_idx, subset_info) => { h.write_all(b"i").unwrap(); h.write_all(&input_idx.0.to_le_bytes()).unwrap(); - if let Some(SubsetInfo(outputs, strict)) = subset_info { + if let Some(SubsetInfo(outputs)) = subset_info { h.write_all(b"ss").unwrap(); for output in &outputs { h.write_all(output.as_bytes()).unwrap(); h.write_all(b",").unwrap(); } - h.write_all(if strict { b"s" } else { b"l" }).unwrap(); + h.write_all(b"l").unwrap(); // TODO: Remove the next time we break spec-hash values } } RuntimeDep(dep_idx, subset_info) => { h.write_all(b"r").unwrap(); h.write_all(&dep_idx.0.to_le_bytes()).unwrap(); - if let Some(SubsetInfo(outputs, strict)) = subset_info { + if let Some(SubsetInfo(outputs)) = subset_info { h.write_all(b"ss").unwrap(); for output in &outputs { h.write_all(output.as_bytes()).unwrap(); h.write_all(b",").unwrap(); } - h.write_all(if strict { b"s" } else { b"l" }).unwrap(); + h.write_all(b"l").unwrap(); // TODO: Remove the next time we break spec-hash values } } ReplaceOnCycle(r_idx) => { @@ -103,9 +103,7 @@ impl<'a> SpecHasher<'a> { let mut edges = Edges::new(); for (bsr, subset_info) in build.inputs.iter().filter_map(|i| match i { BuildSpecInput::Build(bsr) => Some((bsr, None)), - BuildSpecInput::Subset(si) => { - Some((&si.from, Some(SubsetInfo(si.outputs.clone(), si.strict)))) - } + BuildSpecInput::Subset(si) => Some((&si.from, Some(SubsetInfo(si.outputs.clone())))), _ => None, }) { edges.push(Edge::BuildInput(self.process(bsr), subset_info)); @@ -115,7 +113,7 @@ impl<'a> SpecHasher<'a> { RuntimeDep::Build(bsr) => edges.push(Edge::RuntimeDep(self.process(bsr), None)), RuntimeDep::Subset(si) => edges.push(Edge::RuntimeDep( self.process(&si.from), - Some(SubsetInfo(si.outputs.clone(), si.strict)), + Some(SubsetInfo(si.outputs.clone())), )), } } @@ -227,6 +225,47 @@ fn build_attrs_hash(spec: &BuildSpec, h: &mut Hasher) { } } +/// Computes the [SpecHash] for a subset. +pub struct SubsetHasher; + +impl SubsetHasher { + /// Computs the [SpecHash] of a subset of outputs from the given [BuildSpecRef]. + pub fn hash_single<'a, S: Into>( + graph: &'a DepGraph, + bsr: &'a BuildSpecRef, + outputs: Vec, + ) -> SpecHash { + let build = graph.get(bsr).unwrap(); + let outputs: Vec<_> = outputs.into_iter().map(|s| s.into()).collect(); + for output in outputs.iter() { + if build.outputs.get(output).is_none() { + panic!( + "consistency check: cannot form subset over non-existent output {}", + output + ); + } + } + + let subset_spec = SubsetSpec::new_single(&graph.spec_hash(bsr), outputs); + Self::hash(&subset_spec) + } + + /// Computs the [SpecHash] of the given [SubsetSpec]. + pub fn hash(subset_spec: &SubsetSpec) -> SpecHash { + let mut h = Hasher::new(); + h.write_all(b"subset").unwrap(); + for (spec, sorted_outputs) in subset_spec.iter_components() { + h.write_all(spec.as_bytes()).unwrap(); + for output in sorted_outputs { + h.write_all(b"-output").unwrap(); + h.write_all(output.as_bytes()).unwrap(); + } + } + + SpecHash(h.finalize()) + } +} + #[cfg(test)] mod tests { use super::*; @@ -358,4 +397,43 @@ mod tests { .unwrap(), ); } + + #[test] + fn subset_hash() { + let sr = SpecReader::new( + indoc! { + " + let {BuildSpec, HostPath, Source, Prebuilt, OutputLib, OutputBin, OutputData, ..} = import \"minimal.ncl\" in + { + name = \"single buildspec\", + inputs = [ + {url = \"http://uwu.com\", sha256 = \"abcdef\"} | Source, + {package = \"uwu\"} | Prebuilt, + {path = \"/\"} | HostPath, + ], + outputs = { + something = { glob = \"/usr/lib/something.*.so\" } | OutputLib, + uwu_tool = { glob = \"/bin/uwu\" } | OutputBin, + some_data = { glob = \"/data/locale/*\" } | OutputData, + }, + cmd = \"something\", + } | BuildSpec" + } + .to_string(), + &SpecReaderOptions::for_test(), + ) + .unwrap_or_else(|e| { + e.report_to_stderr(); + panic!("spec parsing failed"); + }); + + let dp = DepGraph::new(sr).unwrap(); + + // println!("{}", SpecHasher::hash(&dp, &dp.top_levels[0]).0.to_hex()); + assert_eq!( + SubsetHasher::hash_single(&dp, &dp.top_levels[0], vec!["uwu_tool", "something"]), + SpecHash::from_hex("23d6c828d07f1572f7d8ec315c0a0fb92b00079d290692358cefad45b4b51cc5") + .unwrap(), + ); + } } diff --git a/crates/minimal/Cargo.toml b/crates/minimal/Cargo.toml index 36ecad8e0a..fb0d2c673f 100644 --- a/crates/minimal/Cargo.toml +++ b/crates/minimal/Cargo.toml @@ -53,7 +53,6 @@ docker_credential.workspace = true size.workspace = true object.workspace = true -globset.workspace = true [target.'cfg(target_os = "linux")'.dependencies] hakoniwa.workspace = true diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index f2d92c40ec..05a89ff3f9 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -202,7 +202,7 @@ pub async fn cmd_build_impl( futures::executor::block_on(remote_cache.materialize( &graph.spec_hash(&bsr), - name, + cache::MetaInner::Spec(name.clone()), cache, )) .unwrap(); diff --git a/crates/minimal/src/cmd_check/outputs.rs b/crates/minimal/src/cmd_check/outputs.rs index 5f817b82c9..c70ecac2d6 100644 --- a/crates/minimal/src/cmd_check/outputs.rs +++ b/crates/minimal/src/cmd_check/outputs.rs @@ -1,6 +1,6 @@ use std::{ collections::{HashMap, HashSet}, - path::{Path, PathBuf}, + path::PathBuf, }; use super::{CheckResult, CheckVerdict}; @@ -49,7 +49,7 @@ pub(crate) fn output_types_valid( | BuildOutput::Library { glob } | BuildOutput::Data { glob } => glob, }; - for path in match_paths_for_glob(&cached_build, glob)?.into_iter() { + for path in common::match_files_for_glob(cached_build.path(), glob)?.into_iter() { let data = std::fs::read(&path).map_err(anyhow::Error::from)?; match (object::File::parse(&*data), output) { (Ok(_), BuildOutput::Binary { .. } | BuildOutput::Library { .. }) => {} @@ -86,49 +86,6 @@ pub(crate) fn output_types_valid( Ok(result) } -fn match_paths_for_glob( - cached_build: &DirCacheEntry, - glob: &str, -) -> Result, Error> { - let mut results = Vec::new(); - - let matcher = globset::GlobBuilder::new(glob) - .literal_separator(true) - .empty_alternates(true) - .build() - .map_err(anyhow::Error::from)? - .compile_matcher(); - - // Walk the filesystem starting from root_dir - fn walk_dir( - dir: &Path, - root_dir: &Path, - matcher: &globset::GlobMatcher, - results: &mut Vec, - ) -> Result<(), Error> { - let entries = std::fs::read_dir(dir).map_err(anyhow::Error::from)?; - - for entry in entries { - let entry = entry.map_err(anyhow::Error::from)?; - - let path = entry.path(); - let subset = path.strip_prefix(root_dir).unwrap(); - if path.is_file() && matcher.is_match(subset) { - results.push(path); - } else if path.is_dir() { - // Recursively walk subdirectories - walk_dir(&path, root_dir, matcher, results)?; - } - } - - Ok(()) - } - - let root_dir = cached_build.path(); - walk_dir(root_dir, root_dir, &matcher, &mut results)?; - Ok(results) -} - pub(crate) fn missing_runtime_deps( pkg: &String, all_graph: &Option, @@ -191,7 +148,7 @@ pub(crate) fn missing_runtime_deps( | BuildOutput::Library { glob } | BuildOutput::Data { glob } => glob, }; - for path in match_paths_for_glob(&cached_build, glob)?.into_iter() { + for path in common::match_files_for_glob(cached_build.path(), glob)?.into_iter() { let data = std::fs::read(&path).map_err(anyhow::Error::from)?; if let (Ok(elf), BuildOutput::Binary { .. } | BuildOutput::Library { .. }) = (object::File::parse(&*data), output) @@ -319,7 +276,10 @@ fn find_lib_in_deps( for (i, (_bsr, dep_files)) in deps.iter().enumerate() { let base = dep_files.path(); - if let Some(candidate) = (match_paths_for_glob(dep_files, &glob)?).into_iter().next() { + if let Some(candidate) = (common::match_files_for_glob(dep_files.path(), &glob)?) + .into_iter() + .next() + { return Ok(Some(( i, candidate.strip_prefix(base).unwrap().to_path_buf(), diff --git a/crates/minimal/src/main.rs b/crates/minimal/src/main.rs index 625c364561..ab7434ff16 100644 --- a/crates/minimal/src/main.rs +++ b/crates/minimal/src/main.rs @@ -1,6 +1,7 @@ #![allow(clippy::result_large_err)] use anyhow::Result; +use anyhow::anyhow; use anyhow::bail; use cache::{Cache, LocalDir, RemoteCache, RemoteError}; use clap::{Args, Parser, Subcommand}; @@ -337,6 +338,14 @@ impl From for Error { Self::Other(e) } } +impl From for Error { + fn from(e: common::GlobError) -> Self { + match e { + common::GlobError::IO(e) => Self::Other(e.into()), + common::GlobError::Glob(e) => Self::Other(anyhow!("glob error: {:?}", e.kind())), + } + } +} #[tokio::main] async fn main() -> Result<()> { diff --git a/crates/minimal/src/run.rs b/crates/minimal/src/run.rs index 8a00fdba05..727647a188 100644 --- a/crates/minimal/src/run.rs +++ b/crates/minimal/src/run.rs @@ -4,7 +4,7 @@ use cache::{Cache, LocalDir, MetaInner, PendingDir, RemoteCache}; use common::SpecHash; use google_cloud_storage::client::Storage as GcsStorage; use graph::dep_graph::SourceFetch; -use graph::{BinProvider, ExecPlan}; +use graph::{BinProvider, ExecPlan, RuntimeDep, SubsetInput}; use graph::{BuildOutput, BuildSpec, BuildSpecInput, BuildSpecRef, DepGraph, SourceInput}; use std::collections::HashSet; use std::path::{Path, PathBuf}; @@ -192,7 +192,7 @@ async fn path_for_self_spec( remote_cache .as_ref() .unwrap() - .materialize(input_hash, &input_build.name, cache) + .materialize(input_hash, MetaInner::Spec(input_build.name.clone()), cache) .await?; let cache_path = cache.read_dir(input_hash).unwrap().path().to_path_buf(); Ok((*input_ref, (cache_path, PathBuf::from("/")))) @@ -236,6 +236,124 @@ async fn path_for_self_spec( } } +/// Yields the cached directory containing the subset of build artifacts described by the given [SubsetInput]. +/// +/// If the subset is not already materialized in the cache, it will be created. +pub async fn materialize_subset( + subset: &SubsetInput, + graph: &DepGraph, + cache: &Cache, + remote_cache: Option<&RemoteCache>, +) -> Result<(BuildSpecRef, (PathBuf, PathBuf))> { + let build = graph.get(&subset.from).unwrap(); + let subset_spec = subset.as_spec(graph); + let subset_hash = graph.subset_hash(subset); + + match ( + cache.read_dir(&subset_hash), + remote_cache.as_ref().map(|c| c.exists(&subset_hash)), + ) { + // We have the subset already in the local cache. + (Ok(cache_entry), _) => { + let cache_path = cache_entry.path().to_path_buf(); + Ok((subset.from, (cache_path, PathBuf::from("/")))) + } + // In the remote cache but not the local cache. + (_, Some(true)) => { + let name = format!("{} (subset)", build.name); + let span = tracing::info_span!( + "download_cached", + "indicatif.pb_show" = tracing::field::Empty, + "build" = name, + ); + let _enter = span.enter(); + + remote_cache + .as_ref() + .unwrap() + .materialize(&subset_hash, MetaInner::Subset(subset_spec), cache) + .await?; + let cache_path = cache.read_dir(&subset_hash).unwrap().path().to_path_buf(); + Ok((subset.from, (cache_path, PathBuf::from("/")))) + } + (Err(cache::CacheErr::NotFound), None | Some(false)) => { + // Subset is neither downloadable nor in the local cache. + // Either: + // - We can build it: we have built the thing which this is a subset of + // - We can't (build we are a subset of isnt built) but theres a cycle-breaker. + let dep_hash = graph.spec_hash(&subset.from); + if let Ok(cache_dir) = cache.read_dir(&dep_hash) { + let out = cache.write_dir(&subset_hash).unwrap(); + // Lets build it from the upstream + let mut files: Vec<_> = subset + .outputs + .iter() + .map(|output_name| { + common::match_files_for_glob( + cache_dir.path(), + build.outputs.get(output_name).unwrap().glob(), + ) + }) + .collect::, _>>()? + .into_iter() + .flatten() + .collect(); + files.sort(); + files.dedup(); + + for file in files.into_iter() { + let trimmed = file.strip_prefix(cache_dir.path()).unwrap(); + if let Some(parent) = trimmed.parent() { + std::fs::create_dir_all(out.path().join(parent))?; + } + std::fs::copy(&file, out.path().join(trimmed))?; + } + out.finalize(cache::EntryMeta { + inner: MetaInner::Subset(subset_spec), + fetched: false, + ..Default::default() + }) + .unwrap(); + let cache_path = cache.read_dir(&subset_hash).unwrap().path().to_path_buf(); + return Ok((subset.from, (cache_path, PathBuf::from("/")))); + }; + + match build.replace_on_cycle { + None => panic!( + "missing build in cache for subset of {} [{}]", + build.name, + subset_hash.0.to_hex(), + ), + Some(cycle_breaker) => { + let breaker_hash = graph.spec_hash(&cycle_breaker); + let breaker_build = graph.get(&cycle_breaker).unwrap(); + + debug!( + " --- subbing in cycle breaker {} to resolve subset [{}] ---", + breaker_build.name, + breaker_hash.0.to_hex() + ); + Box::pin(materialize_subset( + &subset.override_build(cycle_breaker), + graph, + cache, + remote_cache, + )) + .await + } + } + } + (Err(e), _) => { + panic!( + "unexpected cache error when resolving path for {} [{}]: {:?}", + build.name, + subset_hash.0.to_hex(), + e + ); + } + } +} + /// yields 'dependencies' mappings for the transitive dependencies of the given build spec. async fn path_transitive_deps_of( input_build: &BuildSpec, @@ -245,15 +363,23 @@ async fn path_transitive_deps_of( remote_cache: Option<&RemoteCache>, ) -> Result<()> { for dep in input_build.runtime_deps.iter() { - let (dep_bsr, dep_paths) = path_for_self_spec( - dep.bsr(), - &graph.spec_hash(dep.bsr()), - graph.get(dep.bsr()).unwrap(), - graph, - cache, - remote_cache, - ) - .await?; + let (dep_bsr, dep_paths) = match dep { + RuntimeDep::Build(bsr) => { + path_for_self_spec( + bsr, + &graph.spec_hash(dep.bsr()), + graph.get(dep.bsr()).unwrap(), + graph, + cache, + remote_cache, + ) + .await? + } + RuntimeDep::Subset(subset) => { + materialize_subset(subset, graph, cache, remote_cache).await? + } + }; + if !out_paths.contains(&dep_paths.0) { let dep_build = graph.get(&dep_bsr).unwrap(); debug!( @@ -403,7 +529,11 @@ impl<'a> Run<'a> { all_paths_for_spec(dep_ref, self.graph, &self.cache, remote_cache).await?; dependencies.extend(input_paths); } - Subset(s) => todo!("subsets: {:?}", s), + Subset(s) => { + let (_based_on_bsr, paths) = + materialize_subset(s, self.graph, &self.cache, remote_cache).await?; + dependencies.insert(paths.0); + } HostPath(_) => { todo!(); } diff --git a/packages/icu/build.ncl b/packages/icu/build.ncl index 10dbf375c0..e8691c723f 100644 --- a/packages/icu/build.ncl +++ b/packages/icu/build.ncl @@ -1,4 +1,4 @@ -let { BuildSpec, Local, OutputBin, OutputData, OutputLib, Source, .. } = import "minimal.ncl" in +let { subsetOf, BuildSpec, Local, OutputBin, OutputData, OutputLib, Source, .. } = import "minimal.ncl" in let base = import "../base/build.ncl" in let gcc = import "../gcc/build.ncl" in let toolchain = import "../toolchain/build.ncl" in @@ -18,7 +18,7 @@ let glibc = import "../glibc/build.ncl" in ], runtime_deps = [ glibc, - gcc, # TODO: We really need to use a subset for this + subsetOf gcc ["libgcc", "libstdcpp"], ], cmd = "./build.sh", From 822958d91bdc6ea262df0791bbfe6e855f8f90f6 Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 14 Oct 2025 19:39:12 -0700 Subject: [PATCH 28/39] crates/graph: make transitive enumeration subset-aware --- crates/graph/src/transitives.rs | 348 +++++++++++++++++++++++++++++--- 1 file changed, 317 insertions(+), 31 deletions(-) diff --git a/crates/graph/src/transitives.rs b/crates/graph/src/transitives.rs index bebaff3232..b406479c7a 100644 --- a/crates/graph/src/transitives.rs +++ b/crates/graph/src/transitives.rs @@ -1,9 +1,25 @@ -use crate::{BuildSpecInput, BuildSpecRef, DepGraph, SubsetInput}; -use std::collections::HashMap; +use crate::{BuildSpecInput, BuildSpecRef, DepGraph, RuntimeDep, SubsetInput}; +use std::collections::{HashMap, HashSet}; -/// Information about why a transitive dependency exists. +/// Information about a dependency. #[derive(Debug, Clone, PartialEq, Eq)] -pub enum DepInfo { +pub struct Dep { + pub needed_by: Vec, + pub outputs: Option>, +} + +impl From> for Dep { + fn from(needed_by: Vec) -> Self { + Self { + needed_by, + outputs: None, + } + } +} + +/// Information about why a transitive dependency exists. +#[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq)] +pub enum Attribution { /// The dependency is declared on the build spec. Ours, /// The dependency is transitively required by a input spec. @@ -16,10 +32,48 @@ pub struct Transitives { pub build: BuildSpecRef, /// The transitive runtime dependencies any use of this build needs, along with attribution. - pub transitive_runtime_deps: HashMap>, + pub transitive_runtime_deps: HashMap, } impl Transitives { + fn upsert, O: IntoIterator>( + transitive_runtime_deps: &mut HashMap, + bsr: &BuildSpecRef, + needed_by: I, + outputs: Option, + ) { + match transitive_runtime_deps.get_mut(bsr) { + Some(existing) => { + // union attributions + existing.needed_by.extend(needed_by); + existing.needed_by.sort(); + existing.needed_by.dedup(); + + // union outputs + match (&mut existing.outputs, outputs) { + (None, _) | (_, None) => { + // One of the outputs wasnt a subset, so we turn into the full build. + existing.outputs = None; + } + (Some(stored), Some(new)) => { + // Both stored & incoming are subsets, union the outputs they are requesting + stored.extend(new); + } + } + } + None => { + // New entry + transitive_runtime_deps.insert( + *bsr, + Dep { + needed_by: needed_by.collect(), + outputs: outputs.map(|v| v.into_iter().collect()), + }, + ); + } + } + } + /// Constructs the set of transitive dependencies for the given build. pub fn new(g: &DepGraph, bsr: &BuildSpecRef, include_inputs: bool) -> Self { let build = g.get(bsr).unwrap(); @@ -32,14 +86,46 @@ impl Transitives { }; for dep in build.runtime_deps.iter() { - out.transitive_runtime_deps - .insert(*dep.bsr(), vec![DepInfo::Ours]); + match dep { + RuntimeDep::Build(bsr) => { + Self::upsert( + &mut out.transitive_runtime_deps, + bsr, + [Attribution::Ours].into_iter(), + None::>, + ); + } + RuntimeDep::Subset(SubsetInput { from: bsr, outputs }) => { + Self::upsert( + &mut out.transitive_runtime_deps, + bsr, + [Attribution::Ours].into_iter(), + Some(outputs.clone()), + ); + } + } } if include_inputs { for dep in build.inputs.iter() { - if let Build(bsr) | Subset(SubsetInput { from: bsr, .. }) = dep { - out.transitive_runtime_deps - .insert(*bsr, vec![DepInfo::Ours]); + use BuildSpecInput::*; + match dep { + Build(bsr) => { + Self::upsert( + &mut out.transitive_runtime_deps, + bsr, + [Attribution::Ours].into_iter(), + None::>, + ); + } + Subset(SubsetInput { from: bsr, outputs }) => { + Self::upsert( + &mut out.transitive_runtime_deps, + bsr, + [Attribution::Ours].into_iter(), + Some(outputs.clone()), + ); + } + Source(_) | Prebuilt(_, _) | Local(_) | HostPath(_) => {} } } } @@ -51,23 +137,33 @@ impl Transitives { .inputs .iter() .filter_map(|input| match (input, include_inputs) { - (Build(bsr) | Subset(SubsetInput { from: bsr, .. }), true) => Some(bsr), + (Build(bsr), true) => Some((bsr, None)), + (Subset(SubsetInput { from: bsr, outputs }), true) => Some((bsr, Some(outputs))), (Build(_) | Subset(_), false) => None, (Source(_) | HostPath(_) | Local(_) | Prebuilt(_, _), _) => None, }) - .chain(build.runtime_deps.iter().map(|dep| dep.bsr())) - .for_each(|bsr| { - for (dep_bsr, source) in Transitives::new(g, bsr, false) + .chain(build.runtime_deps.iter().map(|dep| match dep { + RuntimeDep::Build(bsr) => (bsr, None), + RuntimeDep::Subset(SubsetInput { from: bsr, outputs }) => (bsr, Some(outputs)), + })) + .for_each(|(bsr, _)| { + for (dep_bsr, info) in Transitives::new(g, bsr, false) .transitive_runtime_deps - .keys() - .map(|runtime_dep| (*runtime_dep, DepInfo::Inherited { from: *bsr })) + .into_iter() + .map(|(dep_bsr, mut info)| { + (dep_bsr, { + info.needed_by.clear(); + info.needed_by.push(Attribution::Inherited { from: *bsr }); + info + }) + }) { - match out.transitive_runtime_deps.get_mut(&dep_bsr) { - Some(source_list) => source_list.push(source), - None => { - out.transitive_runtime_deps.insert(dep_bsr, vec![source]); - } - } + Self::upsert( + &mut out.transitive_runtime_deps, + &dep_bsr, + info.needed_by.into_iter(), + info.outputs, + ); } }); @@ -158,7 +254,10 @@ mod tests { .collect::>(), vec![( dg.by_name("runtime dep").next().unwrap(), - vec![DepInfo::Ours] + Dep { + needed_by: vec![Attribution::Ours], + outputs: None, + } )], ); } @@ -220,13 +319,19 @@ mod tests { vec![ ( dg.by_name("nested input").next().unwrap(), - vec![DepInfo::Ours] + Dep { + needed_by: vec![Attribution::Ours], + outputs: None, + } ), ( dg.by_name("runtime dep").next().unwrap(), - vec![DepInfo::Inherited { - from: dg.by_name("nested input").next().unwrap() - }] + Dep { + needed_by: vec![Attribution::Inherited { + from: dg.by_name("nested input").next().unwrap() + }], + outputs: None, + } ) ], ); @@ -289,14 +394,195 @@ mod tests { assert_eq!( deps, vec![ - (dg.by_name("top dep").next().unwrap(), vec![DepInfo::Ours],), + ( + dg.by_name("top dep").next().unwrap(), + Dep { + needed_by: vec![Attribution::Ours], + outputs: None, + } + ), ( dg.by_name("nested dep").next().unwrap(), - vec![DepInfo::Inherited { - from: dg.by_name("top dep").next().unwrap() - }] + Dep { + needed_by: vec![Attribution::Inherited { + from: dg.by_name("top dep").next().unwrap() + }], + outputs: None, + } ) ], ); } + + #[test] + fn subsets_combined() { + let sr = SpecReader::new( + indoc! { + " + let {subsetOf, BuildSpec, Source, OutputData, ..} = import \"minimal.ncl\" in + + let runtime_dep = { + name = \"runtime dep\", + inputs = [ + {url = \"http://uwu.com\", sha256 = \"abcdef\"} | Source, + ], + cmd = \"\", + outputs = { + a = {glob = \"a\"} | OutputData, + b = {glob = \"b\"} | OutputData, + } + } | BuildSpec + in + let collection = { + name = \"collection\", + inputs = [], + cmd = \"\", + runtime_deps = [subsetOf runtime_dep [\"a\"]], + } | BuildSpec + in + + { + name = \"top build\", + inputs = [ + { + name = \"input\", + inputs = [], + cmd = \"\", + } | BuildSpec, + ], + runtime_deps = [ + collection, + subsetOf runtime_dep [\"b\"], + ], + cmd = \"\", + } | BuildSpec" + } + .to_string(), + &SpecReaderOptions::for_test(), + ); + // So we can see the actual error when parsing fails + sr.as_ref().err().into_iter().for_each(|e| { + e.report_to_stderr(); + panic!("spec parsing failed"); + }); + let sr = sr.unwrap(); + + let dg = DepGraph::new(sr).unwrap(); + + let toplevel_manifest = Transitives::new(&dg, &dg.top_levels[0], false); + assert_eq!( + toplevel_manifest.build, + dg.by_name("top build").next().unwrap() + ); + + let mut result = toplevel_manifest + .transitive_runtime_deps + .into_iter() + .collect::>(); + result.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!( + result, + vec![ + ( + dg.by_name("collection").next().unwrap(), + Dep { + needed_by: vec![Attribution::Ours], + outputs: None, + } + ), + ( + dg.by_name("runtime dep").next().unwrap(), + Dep { + needed_by: vec![ + Attribution::Ours, + Attribution::Inherited { + from: dg.by_name("collection").next().unwrap() + } + ], + outputs: Some(["a", "b"].into_iter().map(|s| s.to_string()).collect()), + } + ), + ], + ); + } + + #[test] + fn subset_promoted() { + let sr = SpecReader::new( + indoc! { + " + let {subsetOf, BuildSpec, Source, OutputData, ..} = import \"minimal.ncl\" in + + let runtime_dep = { + name = \"runtime dep\", + inputs = [ + {url = \"http://uwu.com\", sha256 = \"abcdef\"} | Source, + ], + cmd = \"\", + outputs = { + a = {glob = \"a\"} | OutputData, + b = {glob = \"a\"} | OutputData, + } + } | BuildSpec + in + + { + name = \"top build\", + inputs = [ + { + name = \"input\", + inputs = [], + cmd = \"\", + } | BuildSpec, + runtime_dep, + ], + runtime_deps = [ + subsetOf runtime_dep [\"a\"], + ], + cmd = \"\", + } | BuildSpec" + } + .to_string(), + &SpecReaderOptions::for_test(), + ); + // So we can see the actual error when parsing fails + sr.as_ref().err().into_iter().for_each(|e| { + e.report_to_stderr(); + panic!("spec parsing failed"); + }); + let sr = sr.unwrap(); + + let dg = DepGraph::new(sr).unwrap(); + + let toplevel_manifest = Transitives::new(&dg, &dg.top_levels[0], true); + assert_eq!( + toplevel_manifest.build, + dg.by_name("top build").next().unwrap() + ); + + let mut result = toplevel_manifest + .transitive_runtime_deps + .into_iter() + .collect::>(); + result.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!( + result, + vec![ + ( + dg.by_name("input").next().unwrap(), + Dep { + needed_by: vec![Attribution::Ours], + outputs: None, + } + ), + ( + dg.by_name("runtime dep").next().unwrap(), + Dep { + needed_by: vec![Attribution::Ours], + outputs: None, + } + ), + ], + ); + } } From 9570849aaccf4eb100b57314cc67dae2f86ac559 Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 14 Oct 2025 19:55:11 -0700 Subject: [PATCH 29/39] minimal run: add --env, carry TERM,COLORTERM,LS_COLORS vars --- crates/minimal/src/cmd_run.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/minimal/src/cmd_run.rs b/crates/minimal/src/cmd_run.rs index 0a7fd8d593..625b280a4d 100644 --- a/crates/minimal/src/cmd_run.rs +++ b/crates/minimal/src/cmd_run.rs @@ -13,6 +13,10 @@ pub struct RunArgs { #[arg(long, required = false)] rw_dir: Vec, + /// Environment variables to set + #[arg(long, required = false)] + env: Vec, + #[arg(required = true, trailing_var_arg = true, allow_hyphen_values = true)] args: Vec, } @@ -75,6 +79,24 @@ pub async fn cmd_run(args: RunArgs, globals: &GlobalArgs) -> Result<(), Error> { ) .env("LANG", "en_US.utf8") .env("LC_ALL", "en_US.utf8"); + if let Ok(term) = std::env::var("TERM") { + cmd.env("TERM", term.as_str()); + } + if let Ok(ct) = std::env::var("COLORTERM") { + cmd.env("COLORTERM", ct.as_str()); + } + if let Ok(lsc) = std::env::var("LS_COLORS") { + cmd.env("LS_COLORS", lsc.as_str()); + } + for env in args.env { + let mut spl = env.split("="); + let (var, val) = ( + spl.next().unwrap(), + spl.next() + .expect("expected '=' between --env var and value"), + ); + cmd.env(var, val); + } cmd.spawn() .map_err(anyhow::Error::from)? From 6f0f5f36071f08809a4d3f5c382a3a32016a4c2c Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 14 Oct 2025 20:19:07 -0700 Subject: [PATCH 30/39] crates/common: improve error types --- Cargo.lock | 1 - crates/common/Cargo.toml | 1 - crates/common/src/archive.rs | 2 +- crates/common/src/lib.rs | 58 +++++++++++++++++++++++++++++++----- crates/minimal/src/main.rs | 6 ++-- 5 files changed, 54 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf2da861ca..dc3d0501b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -574,7 +574,6 @@ dependencies = [ "smallvec", "tar", "tempfile", - "thiserror 2.0.17", "tracing", "url", "zstd", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 93ab81b33d..4ac44ca187 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -10,7 +10,6 @@ publish.workspace = true indoc.workspace = true [dependencies] -thiserror.workspace = true rustc-hash.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/common/src/archive.rs b/crates/common/src/archive.rs index 39f80ff2c4..876f529563 100644 --- a/crates/common/src/archive.rs +++ b/crates/common/src/archive.rs @@ -87,7 +87,7 @@ pub enum Compression { impl Compression { /// Detect compression format from file extension, if recognized. pub fn from_extension(path: &str) -> Option { - if path.ends_with(".tar.gz") { + if path.ends_with(".tar.gz") || path.ends_with(".tgz") { Some(Compression::Gzip) } else if path.ends_with(".tar.xz") { Some(Compression::Xz) diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 9a6f9314db..d4d59c9599 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -9,11 +9,13 @@ pub mod archive; mod spec_hash; pub use spec_hash::SpecHash; mod subsets; +pub use subsets::SubsetSpec; + use std::{ + fmt, io::Write, path::{Path, PathBuf}, }; -pub use subsets::SubsetSpec; use tracing::warn; /// Implements [Write], mirroring all writes to two underlying writers. @@ -45,14 +47,38 @@ impl Write for Tee { } /// The error produced by calls to [hardlink_dir_contents]. -#[derive(Debug, thiserror::Error)] +#[derive(Debug)] pub enum HardlinkError { - #[error("IO error: {0}")] IO(&'static str, PathBuf, std::io::Error), - #[error("Hardlink error: {0}")] HardlinkFailed(PathBuf, PathBuf, std::io::Error), } +impl fmt::Display for HardlinkError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + HardlinkError::IO(ctx, path, e) => { + write!(f, "{} I/O error at path {}: {}", ctx, path.display(), e) + } + HardlinkError::HardlinkFailed(from, to, e) => write!( + f, + "failed to hardlink {} to {}: {}", + from.display(), + to.display(), + e + ), + } + } +} + +impl std::error::Error for HardlinkError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + HardlinkError::IO(_, _, e) => Some(e), + HardlinkError::HardlinkFailed(_, _, e) => Some(e), + } + } +} + /// Creates a hardlink farm in dst representing files in src, recursively. pub fn hardlink_dir_contents(src: &Path, dst: &Path) -> Result<(), HardlinkError> { use std::fs; @@ -105,14 +131,32 @@ pub fn hardlink_dir_contents(src: &Path, dst: &Path) -> Result<(), HardlinkError Ok(()) } -#[derive(Debug, thiserror::Error)] +#[derive(Debug)] pub enum GlobError { - #[error("IO error: {0}")] IO(std::io::Error), - #[error("Glob error: {0}")] Glob(globset::Error), } +impl fmt::Display for GlobError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + GlobError::IO(e) => { + write!(f, "I/O error: {}", e) + } + GlobError::Glob(e) => write!(f, "glob error: {}", e), + } + } +} + +impl std::error::Error for GlobError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + GlobError::IO(e) => Some(e), + GlobError::Glob(e) => Some(e), + } + } +} + impl From for GlobError { fn from(e: globset::Error) -> Self { GlobError::Glob(e) diff --git a/crates/minimal/src/main.rs b/crates/minimal/src/main.rs index ab7434ff16..1434130273 100644 --- a/crates/minimal/src/main.rs +++ b/crates/minimal/src/main.rs @@ -1,8 +1,6 @@ #![allow(clippy::result_large_err)] -use anyhow::Result; -use anyhow::anyhow; -use anyhow::bail; +use anyhow::{Result, bail}; use cache::{Cache, LocalDir, RemoteCache, RemoteError}; use clap::{Args, Parser, Subcommand}; use google_cloud_storage::{Error as GcsError, client::Storage as GcsStorage}; @@ -342,7 +340,7 @@ impl From for Error { fn from(e: common::GlobError) -> Self { match e { common::GlobError::IO(e) => Self::Other(e.into()), - common::GlobError::Glob(e) => Self::Other(anyhow!("glob error: {:?}", e.kind())), + common::GlobError::Glob(e) => Self::Other(e.into()), } } } From 4d6b945f7e00b3730d11b37f1eb4384ac71d1503 Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 14 Oct 2025 22:39:10 -0700 Subject: [PATCH 31/39] crate/graph: make planner yield exact wiring information --- crates/graph/src/lib.rs | 2 +- crates/graph/src/planner.rs | 279 +++++++++++++++++++++++++++++---- crates/minimal/src/cmd_plan.rs | 23 ++- crates/minimal/src/run.rs | 8 +- 4 files changed, 268 insertions(+), 44 deletions(-) diff --git a/crates/graph/src/lib.rs b/crates/graph/src/lib.rs index f802839798..2909bafe94 100644 --- a/crates/graph/src/lib.rs +++ b/crates/graph/src/lib.rs @@ -214,7 +214,7 @@ pub use dep_graph::{ SubsetInput, }; -mod planner; +pub mod planner; pub use planner::{BinProvider, ExecPlan, PlanErr}; mod transitives; diff --git a/crates/graph/src/planner.rs b/crates/graph/src/planner.rs index 8a77a3f835..1ff9659618 100644 --- a/crates/graph/src/planner.rs +++ b/crates/graph/src/planner.rs @@ -1,6 +1,6 @@ -use crate::{BuildSpecInput, BuildSpecRef, DepGraph, SubsetInput, Transitives}; +use crate::{BuildSpecInput, BuildSpecRef, DepGraph, SubsetInput, Transitives, transitives}; use nickel_lang_core::term::IndexMap; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; /// The status of a build spec / dependency at some moment during planning. #[derive(Debug, Default, Clone, Ord, PartialOrd, PartialEq, Eq)] @@ -96,6 +96,64 @@ impl BuildInfo { } } +/// Describes a specific dependency that should be wired into a build. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Dep { + /// Use the cached build artifacts corresponding to the given bsr. + Cached(BuildSpecRef, Option>), + /// Use the previously-built artifacts specified as a dependency. + Built { + bsr: BuildSpecRef, + /// If the dependency is a cycle-breaker substitution. + cycle_breaker_for: Option, + /// If the build is impure: that is it was built with cycle-breakers. + built_with_breakers: bool, + /// Specific outputs to use instead of the whole build + outputs: Option>, + }, +} + +impl Dep { + /// Returns false if this dependency is a cycle breaker, meaning the build its part of will eventually + /// need to be rebuilt. + pub fn fully_built(&self) -> bool { + use Dep::*; + match self { + Cached(..) => true, + Built { + cycle_breaker_for, .. + } => !cycle_breaker_for.is_some(), + } + } + + fn bsr(&self) -> &BuildSpecRef { + use Dep::*; + match self { + Cached(bsr, _) => bsr, + Built { bsr, .. } => bsr, + } + } + + pub fn cycle_breaker(&self) -> &Option { + use Dep::*; + match self { + Cached(..) => &None, + Built { + cycle_breaker_for, .. + } => cycle_breaker_for, + } + } + + fn with_outputs(mut self, new_outputs: Option>) -> Self { + match &mut self { + Dep::Built { outputs, .. } | Dep::Cached(_, outputs) => { + *outputs = new_outputs; + } + } + self + } +} + /// Types which can tell the planner what build-specs are built & available. pub trait BinProvider: std::fmt::Debug { fn exists(&self, bsr: &BuildSpecRef) -> bool; @@ -262,29 +320,55 @@ impl<'a, BP: BinProvider> ExecPlan<'a, BP> { /// Returns true if the given dependency should be considered built in an earlier phase. fn is_built(&self, dependency: &BuildSpecRef, cycle_breakers_allowed: bool) -> bool { + self.dep_candidate(dependency, cycle_breakers_allowed) + .is_some() + } + + /// Returns something that can be used to satisfy the requested dependency, if possible. + /// + /// While [Dep] can encode subsets, it is never set here. + fn dep_candidate( + &self, + dependency: &BuildSpecRef, + cycle_breakers_allowed: bool, + ) -> Option { if self.bin_provider.exists(dependency) { - return true; + return Some(Dep::Cached(*dependency, None)); } match self.builds.get(dependency) { None => { // queried dependency was not in .builds, this can happen for cycle breakers // which have not been added in to be reachable. Obviously not built in that case - false + None } Some(info) => { if info.state.is_built() { - return true; + return Some(Dep::Built { + bsr: *dependency, + cycle_breaker_for: None, + built_with_breakers: info.state == BuildState::BuiltUsingBreakers, + outputs: None, + }); } if cycle_breakers_allowed { - info.cycle_breaker - .as_ref() - .map(|bsr| self.is_built(bsr, cycle_breakers_allowed)) - .unwrap_or(false) - } else { - false + if let Some(cycle_breaker) = info.cycle_breaker.as_ref() { + if self.is_built(cycle_breaker, cycle_breakers_allowed) { + return Some(Dep::Built { + bsr: *cycle_breaker, + cycle_breaker_for: Some(*dependency), + built_with_breakers: self + .builds + .get(cycle_breaker) + .map(|i| i.state == BuildState::BuiltUsingBreakers) + .unwrap_or(false), + outputs: None, + }); + } + } } + None } } } @@ -443,12 +527,38 @@ pub enum PlanErr { Cycles(Vec>), } +/// Describes a build which should be performed using a given set of dependencies as part of a build plan. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Build { + pub spec: BuildSpecRef, + pub with_deps: Vec, +} + +impl Build { + fn new(bsr: BuildSpecRef, deps: Vec<(Option, transitives::Dep)>) -> Self { + let mut deps = deps + .into_iter() + .map(|(dep, info)| dep.unwrap().with_outputs(info.outputs)) + .collect::>(); + deps.sort_by(|a, b| a.bsr().cmp(b.bsr())); + + Self { + spec: bsr, + with_deps: deps, + } + } + + /// Returns false if any cycle breakers were used as a dependency. + pub fn full_build(&self) -> bool { + self.with_deps.iter().all(|d| d.fully_built()) + } +} + /// The set of build-specs which can be built in parallel with all deps satisfied by /// the planners' associated [BinProvider], or built in a preceeding phase. #[derive(Debug, Clone, PartialEq)] pub struct BuildPhase { - // (bsr, full_build) - pub builds: Vec<(BuildSpecRef, bool)>, + pub builds: Vec, } impl<'a, BP: BinProvider> Iterator for ExecPlan<'a, BP> { @@ -466,7 +576,7 @@ impl<'a, BP: BinProvider> Iterator for ExecPlan<'a, BP> { // (bsr, used_breakers) let mut built_this_phase: HashMap = HashMap::new(); // (bsr, full_build) - let mut met: Vec<(BuildSpecRef, bool)> = Vec::new(); + let mut met: Vec = Vec::new(); // Iterate all reachable builds which have not been fully built. for (candidate, info) in self.builds.iter().filter_map(|(candidate, info)| { @@ -481,14 +591,25 @@ impl<'a, BP: BinProvider> Iterator for ExecPlan<'a, BP> { // Something is ready to be built if all its inputs are built, and all its transitive // runtime deps (and the transitive runtime deps of its inputs) are built. - let (can_build_full, can_build_breakers) = + let deps: Vec<(Option, transitives::Dep)> = Transitives::new(self.graph, candidate, true) .transitive_runtime_deps - .keys() - .fold((true, true), |acc, dep_bsr| { + .into_iter() + .map(|(bsr, info)| (self.dep_candidate(&bsr, true), info)) + .collect(); + + // Determine if all deps are satisfied somehow (can_build_breakers=true), as well + // as determine if all deps are satisfied without cycle-breakers (can_build_full=true). + let (can_build_full, can_build_breakers) = + deps.iter() + .fold((true, true), |acc, (built_candidate, _info)| { ( - acc.0 & self.is_built(dep_bsr, false), - acc.1 & self.is_built(dep_bsr, true), + acc.0 + & built_candidate + .as_ref() + .map(|d| d.fully_built()) + .unwrap_or(false), + acc.1 & built_candidate.is_some(), ) }); @@ -498,7 +619,7 @@ impl<'a, BP: BinProvider> Iterator for ExecPlan<'a, BP> { // All deps are fully satisfied, so we can fully build candidate (true, _, _) => { built_this_phase.insert(*candidate, false); - met.push((*candidate, true)); + met.push(Build::new(*candidate, deps)); } // All deps are satisfied by fully-builts or cycle-breakers (false, true, true) => { @@ -515,7 +636,7 @@ impl<'a, BP: BinProvider> Iterator for ExecPlan<'a, BP> { }; built_this_phase.insert(*candidate, consider_built_with_breakers); - met.push((*candidate, !consider_built_with_breakers)); + met.push(Build::new(*candidate, deps)); } _ => {} } @@ -704,13 +825,32 @@ mod tests { planner.collect::, _>>().unwrap(), vec![ BuildPhase { - builds: vec![(dp.by_name("nested dep").next().unwrap(), true)] + builds: vec![Build { + spec: dp.by_name("nested dep").next().unwrap(), + with_deps: vec![], + },], }, BuildPhase { - builds: vec![(dp.by_name("dep").next().unwrap(), true)] + builds: vec![Build { + spec: dp.by_name("dep").next().unwrap(), + with_deps: vec![Dep::Built { + bsr: dp.by_name("nested dep").next().unwrap(), + cycle_breaker_for: None, + built_with_breakers: false, + outputs: None + },], + },], }, BuildPhase { - builds: vec![(dp.by_name("top").next().unwrap(), true)] + builds: vec![Build { + spec: dp.by_name("top").next().unwrap(), + with_deps: vec![Dep::Built { + bsr: dp.by_name("dep").next().unwrap(), + cycle_breaker_for: None, + built_with_breakers: false, + outputs: None + },], + },], }, ], ); @@ -823,21 +963,64 @@ mod tests { plan, vec![ BuildPhase { - builds: vec![(dp.by_name("breaker dep").next().unwrap(), true)] + builds: vec![Build { + spec: dp.by_name("breaker dep").next().unwrap(), + with_deps: vec![], + },], }, BuildPhase { - builds: vec![(dp.by_name("breaker").next().unwrap(), true)] + builds: vec![Build { + spec: dp.by_name("breaker").next().unwrap(), + with_deps: vec![Dep::Built { + bsr: dp.by_name("breaker dep").next().unwrap(), + cycle_breaker_for: None, + built_with_breakers: false, + outputs: None + },], + },], }, BuildPhase { builds: vec![ - (dp.by_name("top").next().unwrap(), false), - (dp.by_name("self ref").next().unwrap(), false), + Build { + spec: dp.by_name("top").next().unwrap(), + with_deps: vec![Dep::Built { + bsr: dp.by_name("breaker").next().unwrap(), + cycle_breaker_for: Some(dp.by_name("self ref").next().unwrap()), + built_with_breakers: false, + outputs: None + },], + }, + Build { + spec: dp.by_name("self ref").next().unwrap(), + with_deps: vec![Dep::Built { + bsr: dp.by_name("breaker").next().unwrap(), + cycle_breaker_for: Some(dp.by_name("self ref").next().unwrap()), + built_with_breakers: false, + outputs: None + },], + }, ] }, BuildPhase { builds: vec![ - (dp.by_name("top").next().unwrap(), true), - (dp.by_name("self ref").next().unwrap(), true), + Build { + spec: dp.by_name("top").next().unwrap(), + with_deps: vec![Dep::Built { + bsr: dp.by_name("self ref").next().unwrap(), + cycle_breaker_for: None, + built_with_breakers: true, + outputs: None + },], + }, + Build { + spec: dp.by_name("self ref").next().unwrap(), + with_deps: vec![Dep::Built { + bsr: dp.by_name("self ref").next().unwrap(), + cycle_breaker_for: None, + built_with_breakers: true, + outputs: None + },], + }, ] }, ], @@ -913,10 +1096,24 @@ mod tests { planner.collect::, _>>().unwrap(), vec![ BuildPhase { - builds: vec![(dp.by_name("dep").next().unwrap(), true)] + builds: vec![Build { + spec: dp.by_name("dep").next().unwrap(), + with_deps: vec![Dep::Cached( + dp.by_name("nested dep").next().unwrap(), + None + ),], + },], }, BuildPhase { - builds: vec![(dp.by_name("top").next().unwrap(), true)] + builds: vec![Build { + spec: dp.by_name("top").next().unwrap(), + with_deps: vec![Dep::Built { + bsr: dp.by_name("dep").next().unwrap(), + cycle_breaker_for: None, + built_with_breakers: false, + outputs: None + },], + },], }, ], ); @@ -992,10 +1189,24 @@ mod tests { planner.collect::, _>>().unwrap(), vec![ BuildPhase { - builds: vec![(dp.by_name("nested deeper dep").next().unwrap(), true),] + builds: vec![Build { + spec: dp.by_name("nested deeper dep").next().unwrap(), + with_deps: vec![], + },], }, BuildPhase { - builds: vec![(dp.by_name("dep").next().unwrap(), true),] + builds: vec![Build { + spec: dp.by_name("dep").next().unwrap(), + with_deps: vec![ + Dep::Cached(dp.by_name("nested dep").next().unwrap(), None), + Dep::Built { + bsr: dp.by_name("nested deeper dep").next().unwrap(), + cycle_breaker_for: None, + built_with_breakers: false, + outputs: None + }, + ], + },], }, ], ); diff --git a/crates/minimal/src/cmd_plan.rs b/crates/minimal/src/cmd_plan.rs index 3fa857fd59..7fafca692b 100644 --- a/crates/minimal/src/cmd_plan.rs +++ b/crates/minimal/src/cmd_plan.rs @@ -58,18 +58,29 @@ fn print_plan( eprintln!("✓ = Already built, ⚙️ = To be built"); for (i, phase) in plan.enumerate() { eprintln!("Phase {}", i + 1); - for (bsr, do_full_build) in phase?.builds.iter() { - let build = graph.get(bsr).unwrap(); - let bsh = graph.spec_hash(bsr); + for build in phase?.builds.iter() { + let b = graph.get(&build.spec).unwrap(); + let bsh = graph.spec_hash(&build.spec); let is_cached = cache.read_dir(&bsh).is_ok(); let cached_emoji = if is_cached { "✓" } else { "⚙️" }; + let num_cycle_breakers = build + .with_deps + .iter() + .filter_map(|d| d.cycle_breaker().to_owned()) + .count(); eprintln!( - " - {} {} [{}] full_build={}", + " - {} {} [{}] full_build={} deps={} {}", cached_emoji, - build.name, + b.name, bsh.0.to_hex(), - do_full_build + build.full_build(), + build.with_deps.len(), + if num_cycle_breakers > 0 { + format!("({} cycle-breakers)", num_cycle_breakers) + } else { + "".to_string() + } ); } } diff --git a/crates/minimal/src/run.rs b/crates/minimal/src/run.rs index 727647a188..541fe400f9 100644 --- a/crates/minimal/src/run.rs +++ b/crates/minimal/src/run.rs @@ -705,10 +705,12 @@ impl<'a> Run<'a> { ))); rayon::scope(|s| { - for (bsr, full_build) in phase.unwrap().builds.iter() { + for build in phase.unwrap().builds.iter() { + // TODO: Wire the build based on deps, rather than probing whats available + let tokio_runtime = tokio_runtime.clone(); - let bsr = bsr.to_owned(); - let full_build = full_build.to_owned(); + let bsr = build.spec.to_owned(); + let full_build = build.full_build(); let self2 = self2.clone(); let err_bsr = build_which_errored.clone(); let cache_handles = cache_handles.clone(); From 41237d6b49f6e111e69a6a5c0be8bfa0dd43faee Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Wed, 15 Oct 2025 11:44:03 -0700 Subject: [PATCH 32/39] rm special cased events --- .../src/spongebob_subscriber_v2.rs | 67 +------------------ crates/build-events/src/events.rs | 2 +- crates/build-sandbox/src/executor.rs | 45 ++++++------- 3 files changed, 24 insertions(+), 90 deletions(-) diff --git a/crates/build-events-proto/src/spongebob_subscriber_v2.rs b/crates/build-events-proto/src/spongebob_subscriber_v2.rs index 6384b73d3e..a5783ed181 100644 --- a/crates/build-events-proto/src/spongebob_subscriber_v2.rs +++ b/crates/build-events-proto/src/spongebob_subscriber_v2.rs @@ -28,8 +28,6 @@ use tracing::warn; /// ``` pub struct SpongeBobSubscriberV2 { spongebob: std::sync::Arc, - /// Cache of file paths from BuildMetadata events, keyed by "{target_id}/stdout_path" etc. - file_paths: std::sync::Arc>>, } impl SpongeBobSubscriberV2 { @@ -50,9 +48,6 @@ impl SpongeBobSubscriberV2 { pub fn from_client(client: crate::client::SpongeBob) -> Self { Self { spongebob: std::sync::Arc::new(client), - file_paths: std::sync::Arc::new(tokio::sync::Mutex::new( - std::collections::HashMap::new(), - )), } } } @@ -60,67 +55,7 @@ impl SpongeBobSubscriberV2 { #[async_trait] impl BuildEventSubscriber for SpongeBobSubscriberV2 { async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { - use build_events::events::build_event::Event; - - // Handle BuildMetadata events to cache file paths - if let Some(Event::BuildMetadata(metadata)) = &event.event { - let mut file_paths = self.file_paths.lock().await; - for (key, value) in &metadata.metadata { - if key.ends_with("_path") { - file_paths.insert(key.clone(), value.clone()); - } - } - // Don't publish BuildMetadata with file paths to avoid clutter - return Ok(()); - } - - // Handle TargetCompleted events with file uploads - if let Some(Event::TargetCompleted(target_completed)) = &event.event { - let target_id = &target_completed.target_id; - - // Get file paths from cache - let file_paths = self.file_paths.lock().await; - let stdout_key = format!("{}/stdout_path", target_id); - let stderr_key = format!("{}/stderr_path", target_id); - - let stdout_path = file_paths.get(&stdout_key).cloned(); - let stderr_path = file_paths.get(&stderr_key).cloned(); - drop(file_paths); - - // First publish the TargetCompleted event - if let Err(e) = self.spongebob.publish_build_event(event.clone()).await { - warn!("Failed to publish TargetCompleted event to Spongebob: {}", e); - return Err(SubscriberError::Custom(format!( - "Failed to publish TargetCompleted event: {}", - e - ))); - } - - // Then upload files if paths are available - if let Some(stdout_path) = stdout_path { - if let Ok(contents) = std::fs::read(&stdout_path) { - if let Err(e) = self.spongebob.publish_file_created_event(target_id, "stdout", contents).await { - warn!("Failed to upload stdout for target {}: {}", target_id, e); - } - } else { - warn!("Failed to read stdout file: {}", stdout_path); - } - } - - if let Some(stderr_path) = stderr_path { - if let Ok(contents) = std::fs::read(&stderr_path) { - if let Err(e) = self.spongebob.publish_file_created_event(target_id, "stderr", contents).await { - warn!("Failed to upload stderr for target {}: {}", target_id, e); - } - } else { - warn!("Failed to read stderr file: {}", stderr_path); - } - } - - return Ok(()); - } - - // For all other events, just publish them + // Publish all events to SpongeBob match self.spongebob.publish_build_event(event.clone()).await { Ok(()) => { tracing::debug!("Successfully published build event to Spongebob"); diff --git a/crates/build-events/src/events.rs b/crates/build-events/src/events.rs index d8496d5360..bbb4fa6cad 100644 --- a/crates/build-events/src/events.rs +++ b/crates/build-events/src/events.rs @@ -7,7 +7,7 @@ // Re-export proto types as the canonical event types pub use minimal_spongebob_community_neoeinstein_prost::spongebob::v1::{ build_event, ActionCompleted, ActionStarted, BuildEvent, BuildFinished, BuildMetadata, - BuildStarted, TargetCompleted, TargetKind, TargetStarted, + BuildStarted, FileCreated, TargetCompleted, TargetKind, TargetStarted, }; /// Helper function to get current timestamp in milliseconds diff --git a/crates/build-sandbox/src/executor.rs b/crates/build-sandbox/src/executor.rs index b10ac36f5c..c1014e3b5a 100644 --- a/crates/build-sandbox/src/executor.rs +++ b/crates/build-sandbox/src/executor.rs @@ -1,8 +1,7 @@ use build_events::events::{ - build_event, BuildEvent, BuildMetadata, TargetCompleted, TargetKind, TargetStarted, + build_event, BuildEvent, FileCreated, TargetCompleted, TargetKind, TargetStarted, }; use hakoniwa::Container; -use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -108,28 +107,28 @@ impl BuildExecutor { })), }); - // Emit BuildMetadata with file paths so subscriber can upload them - let mut file_metadata = HashMap::new(); - file_metadata.insert( - format!("{}/stdout_path", target_id), - self.build_workspace_dir - .join("stdout") - .to_string_lossy() - .to_string(), - ); - file_metadata.insert( - format!("{}/stderr_path", target_id), - self.build_workspace_dir - .join("stderr") - .to_string_lossy() - .to_string(), - ); + // Emit FileCreated events for stdout and stderr + if let Ok(stdout_contents) = fs::read(self.build_workspace_dir.join("stdout")) { + build_events::event_bus().emit(BuildEvent { + event: Some(build_event::Event::FileCreated(FileCreated { + target_id: target_id.to_string(), + name: "stdout".to_string(), + contents: stdout_contents.into(), + timestamp_millis: current_millis(), + })), + }); + } - build_events::event_bus().emit(BuildEvent { - event: Some(build_event::Event::BuildMetadata(BuildMetadata { - metadata: file_metadata, - })), - }); + if let Ok(stderr_contents) = fs::read(self.build_workspace_dir.join("stderr")) { + build_events::event_bus().emit(BuildEvent { + event: Some(build_event::Event::FileCreated(FileCreated { + target_id: target_id.to_string(), + name: "stderr".to_string(), + contents: stderr_contents.into(), + timestamp_millis: current_millis(), + })), + }); + } // Return the original result execute_result?; From 77be858d6ee7c00f1b00debc0a7645a4c8d57322 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Wed, 15 Oct 2025 12:05:59 -0700 Subject: [PATCH 33/39] more refactor --- build_events.jsonl | 8 -- crates/build-events-proto/Cargo.toml | 12 ++- crates/build-events-proto/src/client.rs | 108 ++++++------------------ crates/build-events/src/lib.rs | 23 +---- crates/minimal/src/cmd_build.rs | 12 +-- crates/minimal/src/main.rs | 3 +- 6 files changed, 35 insertions(+), 131 deletions(-) delete mode 100644 build_events.jsonl diff --git a/build_events.jsonl b/build_events.jsonl deleted file mode 100644 index d4a7c4cea9..0000000000 --- a/build_events.jsonl +++ /dev/null @@ -1,8 +0,0 @@ -{"type":"build_started","invocation_id":"demo-invocation-123","command_line":["minimal","build","//example:target"],"timestamp_millis":1759951982261,"working_directory":"/Users/bweeks/Code/minimal/minpkgs"} -{"type":"target_started","label":"//example:target","target_kind":"binary","timestamp_millis":1759951982363} -{"type":"action_started","action_id":"action-compile-1","label":"//example:target","mnemonic":"RustcCompile","timestamp_millis":1759951982466} -{"type":"action_completed","action_id":"action-compile-1","success":true,"timestamp_millis":1759951982667,"exit_code":0,"stdout":"Compiling example v0.1.0","stderr":null} -{"type":"action_started","action_id":"action-link-1","label":"//example:target","mnemonic":"RustcLink","timestamp_millis":1759951982769} -{"type":"action_completed","action_id":"action-link-1","success":true,"timestamp_millis":1759951982922,"exit_code":0,"stdout":"Finished release [optimized] target(s)","stderr":null} -{"type":"target_completed","label":"//example:target","success":true,"timestamp_millis":1759951983024,"error_message":null,"outputs":["target/release/example"]} -{"type":"build_finished","invocation_id":"demo-invocation-123","success":true,"timestamp_millis":1759951983126,"error_message":null} diff --git a/crates/build-events-proto/Cargo.toml b/crates/build-events-proto/Cargo.toml index a9aa043547..1acca99671 100644 --- a/crates/build-events-proto/Cargo.toml +++ b/crates/build-events-proto/Cargo.toml @@ -7,17 +7,15 @@ license = "MIT OR Apache-2.0" publish.workspace = true [dependencies] +async-trait = "0.1" build-events.workspace = true -tonic = { version = "0.12.3", features = ["tls", "tls-native-roots"] } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014214004-4ceda9b7e17f.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251014214004-4ceda9b7e17f.1", registry = "buf" } +thiserror.workspace = true tokio = { workspace = true, features = ["sync"] } tokio-stream = "0.1" -thiserror.workspace = true +tonic = { version = "0.12.3", features = ["tls", "tls-native-roots"] } tracing.workspace = true -async-trait = "0.1" - -# BSR proto dependencies for canonical spongebob.v1 types -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014214004-4ceda9b7e17f.1", registry = "buf" } -minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251014214004-4ceda9b7e17f.1", registry = "buf" } [dev-dependencies] tempfile.workspace = true diff --git a/crates/build-events-proto/src/client.rs b/crates/build-events-proto/src/client.rs index b497603a19..7b8922b09a 100644 --- a/crates/build-events-proto/src/client.rs +++ b/crates/build-events-proto/src/client.rs @@ -1,7 +1,7 @@ //! SpongeBob gRPC client for streaming build events use crate::proto::{ - build_event, BuildEvent, BuildStarted, FileCreated, OrderedBuildEvent, PublishBuildEventStreamRequest, + build_event, BuildEvent, BuildStarted, OrderedBuildEvent, PublishBuildEventStreamRequest, build_event_service_client::BuildEventServiceClient, }; use std::sync::atomic::{AtomicI64, Ordering}; @@ -35,8 +35,6 @@ pub type Result = std::result::Result; #[derive(Debug, Clone)] pub struct SpongeBob { request_tx: Arc>>, - invocation_id: String, - url: String, sequence_number: Arc, } @@ -44,14 +42,13 @@ impl SpongeBob { /// Create a new SpongeBob client and open streaming connection /// /// This immediately connects to the server and opens a bidirectional - /// stream. The server assigns an invocation ID which is returned in - /// the first response. + /// stream. The server assigns an invocation ID which is logged. /// /// # Example /// ```ignore /// # async fn example() -> Result<(), Box> { /// let spongebob = build_events_proto::SpongeBob::new().await?; - /// println!("Invocation URL: {}", spongebob.url()); + /// // Invocation URL is automatically logged /// # Ok(()) /// # } /// ``` @@ -97,91 +94,51 @@ impl SpongeBob { let response = client.publish_build_event_stream(request_stream).await?; let mut response_stream = response.into_inner(); - // Create oneshot channel for first response - let (first_response_tx, first_response_rx) = tokio::sync::oneshot::channel(); + // Read first response to get invocation ID + let first_response = response_stream + .message() + .await? + .ok_or_else(|| SpongeBobError::Stream("Server closed stream before sending invocation ID".to_string()))?; - // Spawn background task to continuously read responses - // This task reads responses and sends the first one to the oneshot channel - tokio::spawn(async move { - let mut first_response_tx = Some(first_response_tx); + let invocation_id = first_response + .stream_id + .ok_or_else(|| SpongeBobError::Stream("Server did not provide stream ID".to_string()))? + .invocation_id; + // Log invocation URL at start + tracing::info!("https://dash.minimal.farm/invocations/{}", invocation_id); + + // Spawn background task to drain responses + let invocation_id_for_task = invocation_id.clone(); + tokio::spawn(async move { loop { match response_stream.message().await { Ok(Some(response)) => { - // Send first response to oneshot channel - if let Some(tx) = first_response_tx.take() { - let _ = tx.send(response); - } else { - // Log acknowledgment for debugging - tracing::debug!( - sequence_number = response.sequence_number, - "Received acknowledgment from server" - ); - } + tracing::debug!( + sequence_number = response.sequence_number, + "Received acknowledgment from server" + ); } Ok(None) => { - // Stream closed by server - tracing::info!("Response stream closed by server"); + // Stream closed by server - log final URL + tracing::info!("https://dash.minimal.farm/invocations/{}", invocation_id_for_task); break; } Err(e) => { - // Log error but don't crash - allow graceful degradation tracing::warn!("Error reading from response stream: {}", e); + tracing::info!("https://dash.minimal.farm/invocations/{}", invocation_id_for_task); break; } } } }); - // Wait for first response from background task - let first_response = first_response_rx - .await - .map_err(|_| SpongeBobError::Stream("Background task died before first response".to_string()))?; - - let invocation_id = first_response - .stream_id - .ok_or_else(|| SpongeBobError::Stream("Server did not provide stream ID".to_string()))? - .invocation_id; - - let url = format!("https://dash.minimal.farm/invocations/{}", invocation_id); - - tracing::info!(invocation_id = %invocation_id, "Received server-assigned invocation ID"); - Ok(Self { request_tx: Arc::new(Mutex::new(request_tx)), - invocation_id, - url, sequence_number: Arc::new(AtomicI64::new(2)), // Start at 2 (we used 1 for BuildStarted) }) } - /// Publish a FileCreated event to upload a file for a specific target - #[tracing::instrument(skip(self, contents))] - pub async fn publish_file_created_event( - &self, - target_id: &str, - file_name: &str, - contents: Vec, - ) -> Result<()> { - let timestamp_millis = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as i64; - - let file_created = FileCreated { - target_id: target_id.to_string(), - name: file_name.to_string(), - contents: contents.into(), - timestamp_millis, - }; - - let event = BuildEvent { - event: Some(build_event::Event::FileCreated(file_created)), - }; - - self.publish_build_event(event).await - } - /// Publish a build event to Spongebob via the streaming connection /// /// This method is non-blocking - it sends the event to the stream without @@ -222,19 +179,4 @@ impl SpongeBob { drop(self.request_tx); Ok(()) } - - /// Get the server-assigned invocation ID - pub fn invocation_id(&self) -> &str { - &self.invocation_id - } - - /// Get the dashboard URL for this invocation - pub fn url(&self) -> &str { - &self.url - } - - /// Get the resource name for this invocation - pub fn resource_name(&self) -> String { - format!("invocations/{}", self.invocation_id) - } } diff --git a/crates/build-events/src/lib.rs b/crates/build-events/src/lib.rs index 16896f8bd9..65645fd0f1 100644 --- a/crates/build-events/src/lib.rs +++ b/crates/build-events/src/lib.rs @@ -113,12 +113,6 @@ //! - **Bus emission**: Best-effort, never blocks. Slow subscribers may lag and miss events. //! - **Subscriber errors**: Logged via `tracing::warn!`, but don't stop other subscribers. //! - **Backpressure**: Broadcast channel size is configurable; slow subscribers will lag. -//! -//! # Future Extensions -//! -//! This crate provides pure Rust types. For protobuf support, create a separate -//! `build-events-proto` crate that converts these Rust types to protobuf messages -//! for gRPC streaming. pub mod bus; pub mod dispatcher; @@ -136,19 +130,15 @@ pub use subscriber::{BuildEventSubscriber, SubscriberError}; use std::sync::OnceLock; static GLOBAL_EVENT_BUS: OnceLock = OnceLock::new(); -static INVOCATION_ID: OnceLock = OnceLock::new(); -/// Initialize the global event bus and invocation ID +/// Initialize the global event bus /// /// This should be called once at application startup before any events are emitted. /// Panics if called more than once. -pub fn initialize_global_event_bus(invocation_id: String) { +pub fn initialize_global_event_bus() { GLOBAL_EVENT_BUS .set(BuildEventBus::new(10000)) .expect("Global event bus already initialized"); - INVOCATION_ID - .set(invocation_id) - .expect("Invocation ID already initialized"); } /// Get a reference to the global event bus @@ -159,12 +149,3 @@ pub fn event_bus() -> &'static BuildEventBus { .get() .expect("Global event bus not initialized. Call initialize_global_event_bus() first.") } - -/// Get the invocation ID for this build session -/// -/// Panics if the invocation ID has not been initialized via `initialize_global_event_bus`. -pub fn invocation_id() -> &'static str { - INVOCATION_ID - .get() - .expect("Invocation ID not initialized. Call initialize_global_event_bus() first.") -} diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index 05a89ff3f9..78a404c0d8 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -212,12 +212,8 @@ pub async fn cmd_build_impl( } } - // Display build summary with spongebob URL - let spongebob_url = format!( - "https://dash.minimal.farm/invocations/{}", - build_events::invocation_id() - ); - display_build_summary(graph, &cache, globals, &run, Some(&spongebob_url)); + // Display build summary + display_build_summary(graph, &cache, globals, &run); Ok(()) } @@ -228,7 +224,6 @@ fn display_build_summary( cache: &Cache, _globals: &GlobalArgs, _run: &Run, - command_spongebob_url: Option<&str>, ) { info!("Build completed successfully!"); @@ -247,7 +242,4 @@ fn display_build_summary( } } } - if let Some(url) = command_spongebob_url { - info!("{}", url); - }; } diff --git a/crates/minimal/src/main.rs b/crates/minimal/src/main.rs index 1434130273..a8a61c8847 100644 --- a/crates/minimal/src/main.rs +++ b/crates/minimal/src/main.rs @@ -373,10 +373,9 @@ async fn main() -> Result<()> { let spongebob = build_events_proto::SpongeBob::new() .await .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; - let invocation_id = spongebob.invocation_id().to_string(); // Initialize the global event bus - build_events::initialize_global_event_bus(invocation_id); + build_events::initialize_global_event_bus(); // Setup SpongeBob subscriber use build_events::BuildEventDispatcher; From d6d0bbed8289deadb6f297bf5d44957f4004bc47 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Wed, 15 Oct 2025 12:17:18 -0700 Subject: [PATCH 34/39] more refactor --- crates/build-events-proto/src/client.rs | 33 +++++++ crates/build-events-proto/src/lib.rs | 14 +-- .../src/spongebob_subscriber_v2.rs | 91 ------------------- crates/minimal/src/main.rs | 9 +- 4 files changed, 40 insertions(+), 107 deletions(-) delete mode 100644 crates/build-events-proto/src/spongebob_subscriber_v2.rs diff --git a/crates/build-events-proto/src/client.rs b/crates/build-events-proto/src/client.rs index 7b8922b09a..40cf63bf90 100644 --- a/crates/build-events-proto/src/client.rs +++ b/crates/build-events-proto/src/client.rs @@ -4,6 +4,8 @@ use crate::proto::{ build_event, BuildEvent, BuildStarted, OrderedBuildEvent, PublishBuildEventStreamRequest, build_event_service_client::BuildEventServiceClient, }; +use async_trait::async_trait; +use build_events::{BuildEventSubscriber, SubscriberError}; use std::sync::atomic::{AtomicI64, Ordering}; use std::sync::Arc; use tokio::sync::mpsc; @@ -180,3 +182,34 @@ impl SpongeBob { Ok(()) } } + +/// Implement BuildEventSubscriber trait to allow SpongeBob client to be used +/// directly as a subscriber in the build events dispatcher. +#[async_trait] +impl BuildEventSubscriber for SpongeBob { + async fn on_event(&self, event: &BuildEvent) -> std::result::Result<(), SubscriberError> { + // Publish all events to SpongeBob + match self.publish_build_event(event.clone()).await { + Ok(()) => { + tracing::debug!("Successfully published build event to Spongebob"); + Ok(()) + } + Err(e) => { + tracing::warn!("Failed to publish build event to Spongebob: {}", e); + Err(SubscriberError::Custom(format!( + "Failed to publish event: {}", + e + ))) + } + } + } + + fn name(&self) -> &str { + "SpongeBob" + } + + async fn on_close(&self) -> std::result::Result<(), SubscriberError> { + // No cleanup needed - connection will be closed when dropped + Ok(()) + } +} diff --git a/crates/build-events-proto/src/lib.rs b/crates/build-events-proto/src/lib.rs index 3d000486e5..df0ff0a3f9 100644 --- a/crates/build-events-proto/src/lib.rs +++ b/crates/build-events-proto/src/lib.rs @@ -9,33 +9,31 @@ //! This crate extends the pure Rust `build-events` crate with: //! - **Protobuf message definitions** for all build event types (via BSR) //! - **SpongeBob gRPC client** for streaming build events to spongebob.minimal.farm -//! - **SpongeBobSubscriberV2** for forwarding events to SpongeBob service +//! - **BuildEventSubscriber implementation** on SpongeBob for direct integration //! //! # Architecture //! //! ```text -//! Build Events → BuildEventDispatcher → SpongeBobSubscriberV2 → SpongeBob client → spongebob.minimal.farm +//! Build Events → BuildEventDispatcher → SpongeBob client → spongebob.minimal.farm //! ``` //! //! # Example: Sending Events to SpongeBob //! //! ```ignore //! use build_events::{BuildEventBus, BuildEventDispatcher}; -//! use build_events_proto::{SpongeBob, SpongeBobSubscriberV2}; +//! use build_events_proto::SpongeBob; //! //! #[tokio::main] //! async fn main() -> Result<(), Box> { //! // Create SpongeBob client (opens bidirectional stream) //! let spongebob = SpongeBob::new().await?; -//! let invocation_id = spongebob.invocation_id().to_string(); //! //! // Initialize event bus //! let event_bus = BuildEventBus::new(10000); //! -//! // Setup dispatcher with SpongeBob subscriber +//! // Setup dispatcher with SpongeBob as subscriber //! let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); -//! let subscriber = SpongeBobSubscriberV2::from_client(spongebob); -//! dispatcher.add_subscriber(Box::new(subscriber)); +//! dispatcher.add_subscriber(Box::new(spongebob)); //! //! // Start dispatcher in background //! tokio::spawn(async move { @@ -78,8 +76,6 @@ pub mod proto { } pub mod client; -pub mod spongebob_subscriber_v2; // Re-export main types for convenience pub use client::{SpongeBob, SpongeBobError, Result}; -pub use spongebob_subscriber_v2::SpongeBobSubscriberV2; diff --git a/crates/build-events-proto/src/spongebob_subscriber_v2.rs b/crates/build-events-proto/src/spongebob_subscriber_v2.rs deleted file mode 100644 index a5783ed181..0000000000 --- a/crates/build-events-proto/src/spongebob_subscriber_v2.rs +++ /dev/null @@ -1,91 +0,0 @@ -//! Subscriber that sends build events to Spongebob service - -use async_trait::async_trait; -use build_events::{BuildEvent, BuildEventSubscriber, SubscriberError}; -use tracing::warn; - -/// Subscriber that publishes build events to Spongebob service -/// -/// This subscriber uses the `SpongeBob` client to connect to the -/// Spongebob service and publish build events via bidirectional streaming. -/// -/// # Example -/// ```ignore -/// use build_events_proto::{SpongeBob, SpongeBobSubscriberV2}; -/// -/// #[tokio::main] -/// async fn main() -> Result<(), Box> { -/// // Create a spongebob client (opens stream and gets server-assigned ID) -/// let spongebob = SpongeBob::new().await?; -/// -/// // Create subscriber from the client -/// let subscriber = SpongeBobSubscriberV2::from_client(spongebob); -/// -/// // Use with BuildEventDispatcher -/// // dispatcher.add_subscriber(Box::new(subscriber)); -/// Ok(()) -/// } -/// ``` -pub struct SpongeBobSubscriberV2 { - spongebob: std::sync::Arc, -} - -impl SpongeBobSubscriberV2 { - /// Create subscriber from an existing SpongeBob client - /// - /// # Arguments - /// * `client` - An existing SpongeBob client with active streaming connection - /// - /// # Example - /// ```ignore - /// # use build_events_proto::{SpongeBob, SpongeBobSubscriberV2}; - /// # async fn example() -> Result<(), Box> { - /// let spongebob = SpongeBob::new().await?; - /// let subscriber = SpongeBobSubscriberV2::from_client(spongebob); - /// # Ok(()) - /// # } - /// ``` - pub fn from_client(client: crate::client::SpongeBob) -> Self { - Self { - spongebob: std::sync::Arc::new(client), - } - } -} - -#[async_trait] -impl BuildEventSubscriber for SpongeBobSubscriberV2 { - async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { - // Publish all events to SpongeBob - match self.spongebob.publish_build_event(event.clone()).await { - Ok(()) => { - tracing::debug!("Successfully published build event to Spongebob"); - Ok(()) - } - Err(e) => { - warn!("Failed to publish build event to Spongebob: {}", e); - Err(SubscriberError::Custom(format!( - "Failed to publish event: {}", - e - ))) - } - } - } - - fn name(&self) -> &str { - "SpongeBobSubscriberV2" - } - - async fn on_close(&self) -> Result<(), SubscriberError> { - // No cleanup needed - Ok(()) - } -} - -#[cfg(test)] -mod tests { - #[test] - fn test_subscriber_creation() { - // This test would require a mock SpongeBobInvocation - // For now, just ensure the module compiles - } -} diff --git a/crates/minimal/src/main.rs b/crates/minimal/src/main.rs index a8a61c8847..3a5e017ff9 100644 --- a/crates/minimal/src/main.rs +++ b/crates/minimal/src/main.rs @@ -377,13 +377,8 @@ async fn main() -> Result<()> { // Initialize the global event bus build_events::initialize_global_event_bus(); - // Setup SpongeBob subscriber - use build_events::BuildEventDispatcher; - use build_events_proto::SpongeBobSubscriberV2; - - let mut dispatcher = BuildEventDispatcher::new(build_events::event_bus().subscribe()); - let subscriber = SpongeBobSubscriberV2::from_client(spongebob); - dispatcher.add_subscriber(Box::new(subscriber)); + let mut dispatcher = build_events::BuildEventDispatcher::new(build_events::event_bus().subscribe()); + dispatcher.add_subscriber(Box::new(spongebob)); // Spawn dispatcher in background tokio::spawn(async move { From 4369f556cf28486d09dff10bf271b985d07b66b4 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Wed, 15 Oct 2025 13:01:11 -0700 Subject: [PATCH 35/39] simplify --- Cargo.lock | 11 +- crates/build-events-proto/Cargo.toml | 5 +- crates/build-events-proto/src/client.rs | 144 ++++++++---------------- crates/build-events/Cargo.toml | 2 +- 4 files changed, 59 insertions(+), 103 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc3d0501b2..5415782a5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -371,6 +371,7 @@ dependencies = [ "tonic 0.12.3", "tracing", "tracing-subscriber", + "uuid", ] [[package]] @@ -2221,9 +2222,9 @@ dependencies = [ [[package]] name = "minimal_spongebob_community_neoeinstein-prost" -version = "0.4.0-20251014214004-4ceda9b7e17f.1" +version = "0.4.0-20251015195841-61cd123d44d0.1" source = "sparse+https://buf.build/gen/cargo/" -checksum = "54c207aab4a172c7ffbe80866557cd86a2d2dc1b8672cff5a5165955dd578694" +checksum = "f6ea00e7e69ce45575508fdc1a97a1e901304062f012d8068367cf0e1d7fbfb7" dependencies = [ "prost 0.13.5", "prost-types 0.13.5", @@ -2231,9 +2232,9 @@ dependencies = [ [[package]] name = "minimal_spongebob_community_neoeinstein-tonic" -version = "0.4.1-20251014214004-4ceda9b7e17f.1" +version = "0.4.1-20251015195841-61cd123d44d0.1" source = "sparse+https://buf.build/gen/cargo/" -checksum = "088a8ec4ad0d378cb9d24da98613b0f25c48ffd89fa28f7428a7be2ecce89946" +checksum = "91c1f93a6c1bcd3e9019972149d8a64308460516ea7535a44bcde834047574f1" dependencies = [ "minimal_spongebob_community_neoeinstein-prost", "tonic 0.12.3", @@ -4365,6 +4366,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ "getrandom 0.3.4", + "js-sys", + "wasm-bindgen", ] [[package]] diff --git a/crates/build-events-proto/Cargo.toml b/crates/build-events-proto/Cargo.toml index 1acca99671..b6ed33dbda 100644 --- a/crates/build-events-proto/Cargo.toml +++ b/crates/build-events-proto/Cargo.toml @@ -9,13 +9,14 @@ publish.workspace = true [dependencies] async-trait = "0.1" build-events.workspace = true -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014214004-4ceda9b7e17f.1", registry = "buf" } -minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251014214004-4ceda9b7e17f.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251015195841-61cd123d44d0.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-tonic = { version = "=0.4.1-20251015195841-61cd123d44d0.1", registry = "buf" } thiserror.workspace = true tokio = { workspace = true, features = ["sync"] } tokio-stream = "0.1" tonic = { version = "0.12.3", features = ["tls", "tls-native-roots"] } tracing.workspace = true +uuid = { version = "1.11", features = ["v4"] } [dev-dependencies] tempfile.workspace = true diff --git a/crates/build-events-proto/src/client.rs b/crates/build-events-proto/src/client.rs index 40cf63bf90..e5dcab1921 100644 --- a/crates/build-events-proto/src/client.rs +++ b/crates/build-events-proto/src/client.rs @@ -1,16 +1,14 @@ //! SpongeBob gRPC client for streaming build events use crate::proto::{ - build_event, BuildEvent, BuildStarted, OrderedBuildEvent, PublishBuildEventStreamRequest, - build_event_service_client::BuildEventServiceClient, + build_event, BuildEvent, BuildStarted, build_event_service_client::BuildEventServiceClient, }; use async_trait::async_trait; use build_events::{BuildEventSubscriber, SubscriberError}; -use std::sync::atomic::{AtomicI64, Ordering}; use std::sync::Arc; use tokio::sync::mpsc; -use tokio::sync::Mutex; use tokio_stream::wrappers::ReceiverStream; +use uuid::Uuid; const ENDPOINT: &str = "https://spongebob.minimal.farm"; @@ -26,25 +24,22 @@ pub enum SpongeBobError { pub type Result = std::result::Result; -/// SpongeBob client with server-assigned invocation ID +/// SpongeBob client with client-generated invocation ID /// -/// Creates a bidirectional stream on initialization and receives -/// a server-assigned invocation ID. All events are published through -/// this stream with monotonically increasing sequence numbers. -/// -/// The client spawns a background task to continuously read from the -/// response stream, allowing non-blocking event publishing. +/// Creates a client streaming connection with a client-generated +/// invocation ID. All events are published through this stream. +/// The server sends an Empty response when the stream closes. #[derive(Debug, Clone)] pub struct SpongeBob { - request_tx: Arc>>, - sequence_number: Arc, + request_tx: mpsc::Sender, + invocation_id: Arc, } impl SpongeBob { /// Create a new SpongeBob client and open streaming connection /// - /// This immediately connects to the server and opens a bidirectional - /// stream. The server assigns an invocation ID which is logged. + /// This immediately connects to the server and opens a client + /// streaming connection. The client generates an invocation ID which is logged. /// /// # Example /// ```ignore @@ -56,129 +51,86 @@ impl SpongeBob { /// ``` #[tracing::instrument] pub async fn new() -> Result { + // Generate invocation ID on client side + let invocation_id = Uuid::new_v4().to_string(); + // Connect to service let mut client = BuildEventServiceClient::connect(ENDPOINT).await?; - // Create channel for bidirectional streaming + // Create channel for client streaming let (request_tx, request_rx) = mpsc::channel(100); - // Send initial BuildStarted event to initiate handshake - // Server will respond with assigned invocation ID + // Send initial BuildStarted event with client-generated invocation ID let timestamp_millis = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_millis() as i64; request_tx - .send(PublishBuildEventStreamRequest { - ordered_event: Some(OrderedBuildEvent { - sequence_number: 1, - stream_id: None, // Server will assign - event: Some(BuildEvent { - event: Some(build_event::Event::BuildStarted(BuildStarted { - command: std::env::args().collect::>().join(" "), - timestamp_millis, - working_directory: std::env::current_dir() - .ok() - .and_then(|p| p.to_str().map(|s| s.to_string())) - .unwrap_or_default(), - })), - }), - }), + .send(BuildEvent { + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: invocation_id.clone(), + command: std::env::args().collect::>().join(" "), + timestamp_millis, + working_directory: std::env::current_dir() + .ok() + .and_then(|p| p.to_str().map(|s| s.to_string())) + .unwrap_or_default(), + })), }) .await .map_err(|e| SpongeBobError::Stream(format!("Failed to queue initial event: {}", e)))?; - // Wrap receiver in a stream (initial BuildStarted is already queued) - let request_stream = ReceiverStream::new(request_rx); - - // Open bidirectional stream - this will immediately have data to send - let response = client.publish_build_event_stream(request_stream).await?; - let mut response_stream = response.into_inner(); - - // Read first response to get invocation ID - let first_response = response_stream - .message() - .await? - .ok_or_else(|| SpongeBobError::Stream("Server closed stream before sending invocation ID".to_string()))?; - - let invocation_id = first_response - .stream_id - .ok_or_else(|| SpongeBobError::Stream("Server did not provide stream ID".to_string()))? - .invocation_id; - // Log invocation URL at start tracing::info!("https://dash.minimal.farm/invocations/{}", invocation_id); - // Spawn background task to drain responses + // Spawn task to handle stream and wait for response let invocation_id_for_task = invocation_id.clone(); tokio::spawn(async move { - loop { - match response_stream.message().await { - Ok(Some(response)) => { - tracing::debug!( - sequence_number = response.sequence_number, - "Received acknowledgment from server" - ); - } - Ok(None) => { - // Stream closed by server - log final URL - tracing::info!("https://dash.minimal.farm/invocations/{}", invocation_id_for_task); - break; - } - Err(e) => { - tracing::warn!("Error reading from response stream: {}", e); - tracing::info!("https://dash.minimal.farm/invocations/{}", invocation_id_for_task); - break; - } + let request_stream = ReceiverStream::new(request_rx); + match client.publish_build_event_stream(request_stream).await { + Ok(_response) => { + tracing::info!("Stream closed successfully"); + tracing::info!("https://dash.minimal.farm/invocations/{}", invocation_id_for_task); + } + Err(e) => { + tracing::error!("Stream error: {}", e); + tracing::info!("https://dash.minimal.farm/invocations/{}", invocation_id_for_task); } } }); Ok(Self { - request_tx: Arc::new(Mutex::new(request_tx)), - sequence_number: Arc::new(AtomicI64::new(2)), // Start at 2 (we used 1 for BuildStarted) + request_tx, + invocation_id: Arc::new(invocation_id), }) } /// Publish a build event to Spongebob via the streaming connection /// - /// This method is non-blocking - it sends the event to the stream without - /// waiting for acknowledgment. Acknowledgments are processed asynchronously - /// by a background task spawned in `new()`. + /// This method is non-blocking - it sends the event to the channel. + /// The background task handles sending to the server. #[tracing::instrument(skip(self, event))] pub async fn publish_build_event(&self, event: BuildEvent) -> Result<()> { - let seq = self.sequence_number.fetch_add(1, Ordering::SeqCst); - - // Send event with sequence number - let request = PublishBuildEventStreamRequest { - ordered_event: Some(OrderedBuildEvent { - sequence_number: seq, - stream_id: None, // Only needed on first message - event: Some(event), - }), - }; - - let request_tx = self.request_tx.lock().await; - request_tx - .send(request) + self.request_tx + .send(event) .await .map_err(|e| SpongeBobError::Stream(format!("Failed to send event: {}", e)))?; - tracing::debug!(sequence_number = seq, "Event sent to stream"); + tracing::debug!("Event queued for sending"); Ok(()) } - /// Close the stream (drops the connection) + /// Close the stream and wait for final response /// - /// If BuildFinished was not sent, the server will mark the invocation - /// as terminated. The background task reading responses will detect - /// the closed stream and terminate gracefully. + /// Drops the request sender which closes the stream. The server will + /// process all events and send a final response. The background task + /// will log the final status. pub async fn close(self) -> Result<()> { - // Drop the request sender which will close the stream - // The background task will detect the closed stream and terminate + // Drop the request sender which closes the stream drop(self.request_tx); + // Background task will receive the final response and log it Ok(()) } } diff --git a/crates/build-events/Cargo.toml b/crates/build-events/Cargo.toml index 3adf2d6f81..5477304f82 100644 --- a/crates/build-events/Cargo.toml +++ b/crates/build-events/Cargo.toml @@ -15,7 +15,7 @@ tokio.workspace = true tracing.workspace = true # Proto types as single source of truth -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251014214004-4ceda9b7e17f.1", registry = "buf" } +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251015195841-61cd123d44d0.1", registry = "buf" } [dev-dependencies] serde_json.workspace = true From 72925c4154c26c9c5553f994924a6e6e22550e15 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Wed, 15 Oct 2025 13:25:17 -0700 Subject: [PATCH 36/39] get working again --- Cargo.lock | 5 +- crates/build-events-proto/src/client.rs | 59 ++++++------------- crates/build-events-proto/src/lib.rs | 9 ++- crates/build-events/src/bus.rs | 7 --- crates/build-events/src/dispatcher.rs | 5 +- crates/build-events/src/events.rs | 5 +- crates/build-events/src/lib.rs | 21 ++++++- crates/build-events/src/subscriber.rs | 3 +- .../build-events/src/subscribers/json_file.rs | 21 +++---- crates/build-events/src/subscribers/logger.rs | 5 +- crates/build-events/tests/integration_test.rs | 16 +---- crates/build-sandbox/src/executor.rs | 4 +- crates/build-sandbox/src/executor_stub.rs | 12 +--- crates/build-sandbox/src/lib.rs | 5 +- crates/minimal/Cargo.toml | 1 + crates/minimal/src/cmd_build.rs | 35 ++++++----- crates/minimal/src/main.rs | 18 ++++-- crates/minimal/src/run.rs | 7 ++- 18 files changed, 107 insertions(+), 131 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5415782a5d..c26e5a5127 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2218,6 +2218,7 @@ dependencies = [ "tracing-indicatif", "tracing-subscriber", "url", + "uuid", ] [[package]] @@ -2814,7 +2815,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn", @@ -2827,7 +2828,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn", diff --git a/crates/build-events-proto/src/client.rs b/crates/build-events-proto/src/client.rs index e5dcab1921..def0060e0a 100644 --- a/crates/build-events-proto/src/client.rs +++ b/crates/build-events-proto/src/client.rs @@ -1,14 +1,10 @@ //! SpongeBob gRPC client for streaming build events -use crate::proto::{ - build_event, BuildEvent, BuildStarted, build_event_service_client::BuildEventServiceClient, -}; +use crate::proto::{BuildEvent, build_event_service_client::BuildEventServiceClient}; use async_trait::async_trait; use build_events::{BuildEventSubscriber, SubscriberError}; -use std::sync::Arc; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; -use uuid::Uuid; const ENDPOINT: &str = "https://spongebob.minimal.farm"; @@ -24,63 +20,39 @@ pub enum SpongeBobError { pub type Result = std::result::Result; -/// SpongeBob client with client-generated invocation ID +/// SpongeBob client for streaming build events /// -/// Creates a client streaming connection with a client-generated -/// invocation ID. All events are published through this stream. +/// Creates a client streaming connection for publishing build events. +/// All events are published through this stream. /// The server sends an Empty response when the stream closes. #[derive(Debug, Clone)] pub struct SpongeBob { request_tx: mpsc::Sender, - invocation_id: Arc, } impl SpongeBob { /// Create a new SpongeBob client and open streaming connection /// /// This immediately connects to the server and opens a client - /// streaming connection. The client generates an invocation ID which is logged. + /// streaming connection. The provided invocation ID is logged for tracking. /// /// # Example /// ```ignore /// # async fn example() -> Result<(), Box> { - /// let spongebob = build_events_proto::SpongeBob::new().await?; + /// let invocation_id = uuid::Uuid::new_v4().to_string(); + /// let spongebob = build_events_proto::SpongeBob::new(invocation_id).await?; /// // Invocation URL is automatically logged /// # Ok(()) /// # } /// ``` #[tracing::instrument] - pub async fn new() -> Result { - // Generate invocation ID on client side - let invocation_id = Uuid::new_v4().to_string(); - + pub async fn new(invocation_id: String) -> Result { // Connect to service let mut client = BuildEventServiceClient::connect(ENDPOINT).await?; // Create channel for client streaming let (request_tx, request_rx) = mpsc::channel(100); - // Send initial BuildStarted event with client-generated invocation ID - let timestamp_millis = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as i64; - - request_tx - .send(BuildEvent { - event: Some(build_event::Event::BuildStarted(BuildStarted { - invocation_id: invocation_id.clone(), - command: std::env::args().collect::>().join(" "), - timestamp_millis, - working_directory: std::env::current_dir() - .ok() - .and_then(|p| p.to_str().map(|s| s.to_string())) - .unwrap_or_default(), - })), - }) - .await - .map_err(|e| SpongeBobError::Stream(format!("Failed to queue initial event: {}", e)))?; - // Log invocation URL at start tracing::info!("https://dash.minimal.farm/invocations/{}", invocation_id); @@ -91,19 +63,22 @@ impl SpongeBob { match client.publish_build_event_stream(request_stream).await { Ok(_response) => { tracing::info!("Stream closed successfully"); - tracing::info!("https://dash.minimal.farm/invocations/{}", invocation_id_for_task); + tracing::info!( + "https://dash.minimal.farm/invocations/{}", + invocation_id_for_task + ); } Err(e) => { tracing::error!("Stream error: {}", e); - tracing::info!("https://dash.minimal.farm/invocations/{}", invocation_id_for_task); + tracing::info!( + "https://dash.minimal.farm/invocations/{}", + invocation_id_for_task + ); } } }); - Ok(Self { - request_tx, - invocation_id: Arc::new(invocation_id), - }) + Ok(Self { request_tx }) } /// Publish a build event to Spongebob via the streaming connection diff --git a/crates/build-events-proto/src/lib.rs b/crates/build-events-proto/src/lib.rs index df0ff0a3f9..fcdeb9b3bf 100644 --- a/crates/build-events-proto/src/lib.rs +++ b/crates/build-events-proto/src/lib.rs @@ -22,11 +22,15 @@ //! ```ignore //! use build_events::{BuildEventBus, BuildEventDispatcher}; //! use build_events_proto::SpongeBob; +//! use uuid::Uuid; //! //! #[tokio::main] //! async fn main() -> Result<(), Box> { +//! // Generate invocation ID +//! let invocation_id = Uuid::new_v4().to_string(); +//! //! // Create SpongeBob client (opens bidirectional stream) -//! let spongebob = SpongeBob::new().await?; +//! let spongebob = SpongeBob::new(invocation_id).await?; //! //! // Initialize event bus //! let event_bus = BuildEventBus::new(10000); @@ -57,6 +61,7 @@ //! // Create proto event (build-events types ARE proto types) //! let event = BuildEvent { //! event: Some(build_event::Event::BuildStarted(BuildStarted { +//! invocation_id: "invocation-123".to_string(), //! command: "build".to_string(), //! timestamp_millis: current_millis(), //! working_directory: "/tmp".to_string(), @@ -78,4 +83,4 @@ pub mod proto { pub mod client; // Re-export main types for convenience -pub use client::{SpongeBob, SpongeBobError, Result}; +pub use client::{Result, SpongeBob, SpongeBobError}; diff --git a/crates/build-events/src/bus.rs b/crates/build-events/src/bus.rs index 856216d6ba..8d1aedec98 100644 --- a/crates/build-events/src/bus.rs +++ b/crates/build-events/src/bus.rs @@ -23,7 +23,6 @@ use tokio::sync::broadcast; /// let mut receiver = bus.subscribe(); /// /// bus.emit(BuildEvent { -/// invocation_id: "test-123".to_string(), /// event: Some(build_event::Event::BuildStarted(BuildStarted { /// invocation_id: "test-123".to_string(), /// command: "build".to_string(), @@ -80,7 +79,6 @@ impl BuildEventBus { /// /// let bus = BuildEventBus::new(1000); /// bus.emit(BuildEvent { - /// invocation_id: "test-123".to_string(), /// event: Some(build_event::Event::BuildStarted(BuildStarted { /// invocation_id: "test-123".to_string(), /// command: "build".to_string(), @@ -144,7 +142,6 @@ mod tests { let mut receiver = bus.subscribe(); let event = BuildEvent { - invocation_id: "test-123".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "test-123".to_string(), command: "build".to_string(), @@ -168,7 +165,6 @@ mod tests { let mut receiver2 = bus.subscribe(); let event = BuildEvent { - invocation_id: "test-456".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "test-456".to_string(), command: "build".to_string(), @@ -194,7 +190,6 @@ mod tests { let mut receiver = bus.subscribe(); let event = BuildEvent { - invocation_id: "test-789".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "test-789".to_string(), command: "build".to_string(), @@ -229,7 +224,6 @@ mod tests { let bus = BuildEventBus::new(100); let event1 = BuildEvent { - invocation_id: "early".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "early".to_string(), command: "build".to_string(), @@ -245,7 +239,6 @@ mod tests { let mut receiver = bus.subscribe(); let event2 = BuildEvent { - invocation_id: "late".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "late".to_string(), command: "build".to_string(), diff --git a/crates/build-events/src/dispatcher.rs b/crates/build-events/src/dispatcher.rs index 7b5b16c2ea..cb79253264 100644 --- a/crates/build-events/src/dispatcher.rs +++ b/crates/build-events/src/dispatcher.rs @@ -148,7 +148,7 @@ impl BuildEventDispatcher { #[cfg(test)] mod tests { use super::*; - use crate::events::{build_event, BuildStarted, current_millis}; + use crate::events::{BuildStarted, build_event, current_millis}; use crate::subscriber::SubscriberError; use async_trait::async_trait; use std::sync::Arc; @@ -191,7 +191,6 @@ mod tests { for i in 0..5 { sender .send(BuildEvent { - invocation_id: format!("test-{}", i), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: format!("test-{}", i), command: "build".to_string(), @@ -238,7 +237,6 @@ mod tests { for i in 0..3 { sender .send(BuildEvent { - invocation_id: format!("test-{}", i), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: format!("test-{}", i), command: "build".to_string(), @@ -295,7 +293,6 @@ mod tests { // Send event sender .send(BuildEvent { - invocation_id: "test".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "test".to_string(), command: "build".to_string(), diff --git a/crates/build-events/src/events.rs b/crates/build-events/src/events.rs index bbb4fa6cad..5aaa9f24c6 100644 --- a/crates/build-events/src/events.rs +++ b/crates/build-events/src/events.rs @@ -6,8 +6,8 @@ // Re-export proto types as the canonical event types pub use minimal_spongebob_community_neoeinstein_prost::spongebob::v1::{ - build_event, ActionCompleted, ActionStarted, BuildEvent, BuildFinished, BuildMetadata, - BuildStarted, FileCreated, TargetCompleted, TargetKind, TargetStarted, + ActionCompleted, ActionStarted, BuildEvent, BuildFinished, BuildMetadata, BuildStarted, + FileCreated, TargetCompleted, TargetKind, TargetStarted, build_event, }; /// Helper function to get current timestamp in milliseconds @@ -42,6 +42,7 @@ mod tests { fn test_event_construction() { let event = BuildEvent { event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test-invocation-id".to_string(), command: "build".to_string(), timestamp_millis: 1234567890, working_directory: "/tmp".to_string(), diff --git a/crates/build-events/src/lib.rs b/crates/build-events/src/lib.rs index 65645fd0f1..3ff1256d0f 100644 --- a/crates/build-events/src/lib.rs +++ b/crates/build-events/src/lib.rs @@ -53,7 +53,6 @@ //! //! // Emit events from build //! event_bus.emit(BuildEvent { -//! invocation_id: "build-123".to_string(), //! event: Some(build_event::Event::BuildStarted(BuildStarted { //! invocation_id: "build-123".to_string(), //! command: "cargo build".to_string(), @@ -130,6 +129,7 @@ pub use subscriber::{BuildEventSubscriber, SubscriberError}; use std::sync::OnceLock; static GLOBAL_EVENT_BUS: OnceLock = OnceLock::new(); +static INVOCATION_ID: OnceLock = OnceLock::new(); /// Initialize the global event bus /// @@ -149,3 +149,22 @@ pub fn event_bus() -> &'static BuildEventBus { .get() .expect("Global event bus not initialized. Call initialize_global_event_bus() first.") } + +/// Initialize the global invocation ID +/// +/// This should be called once at application startup before any events are emitted. +/// Panics if called more than once. +pub fn initialize_invocation_id(id: String) { + INVOCATION_ID + .set(id) + .expect("Global invocation ID already initialized"); +} + +/// Get a reference to the global invocation ID +/// +/// Panics if the global invocation ID has not been initialized via `initialize_invocation_id`. +pub fn invocation_id() -> &'static str { + INVOCATION_ID + .get() + .expect("Global invocation ID not initialized. Call initialize_invocation_id() first.") +} diff --git a/crates/build-events/src/subscriber.rs b/crates/build-events/src/subscriber.rs index f1a4c705c3..80f93b76fa 100644 --- a/crates/build-events/src/subscriber.rs +++ b/crates/build-events/src/subscriber.rs @@ -85,7 +85,7 @@ pub trait BuildEventSubscriber: Send + Sync { #[cfg(test)] mod tests { use super::*; - use crate::events::{build_event, BuildStarted, current_millis}; + use crate::events::{BuildStarted, build_event, current_millis}; struct TestSubscriber { name: String, @@ -111,7 +111,6 @@ mod tests { assert_eq!(subscriber.name(), "test"); let event = BuildEvent { - invocation_id: "test-123".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "test-123".to_string(), command: "build".to_string(), diff --git a/crates/build-events/src/subscribers/json_file.rs b/crates/build-events/src/subscribers/json_file.rs index 1cb6413812..9cc579bfa3 100644 --- a/crates/build-events/src/subscribers/json_file.rs +++ b/crates/build-events/src/subscribers/json_file.rs @@ -2,7 +2,7 @@ //! //! Writes build events to a file in JSON-lines format (one JSON object per line). -use crate::events::{build_event, BuildEvent, TargetKind}; +use crate::events::{BuildEvent, TargetKind, build_event}; use crate::subscriber::{BuildEventSubscriber, SubscriberError}; use async_trait::async_trait; use std::io::Write; @@ -69,11 +69,12 @@ impl std::fmt::Debug for JsonFileWriter { /// Convert proto BuildEvent to JSON value (custom serialization since proto types don't have serde) fn event_to_json(event: &BuildEvent) -> serde_json::Value { - let event_type_json = match &event.event { + match &event.event { Some(build_event::Event::BuildStarted(e)) => { serde_json::json!({ "type": "build_started", "build_started": { + "invocation_id": e.invocation_id, "command": e.command, "timestamp_millis": e.timestamp_millis, "working_directory": e.working_directory, @@ -169,9 +170,7 @@ fn event_to_json(event: &BuildEvent) -> serde_json::Value { "type": "unknown", }) } - }; - - event_type_json + } } #[async_trait] @@ -216,7 +215,6 @@ mod tests { assert_eq!(writer.path(), file_path.as_path()); let event = BuildEvent { - invocation_id: "test-123".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "test-123".to_string(), command: "build".to_string(), @@ -240,8 +238,8 @@ mod tests { assert_eq!(lines.len(), 1); let parsed: serde_json::Value = serde_json::from_str(&lines[0]).unwrap(); - assert_eq!(parsed["invocation_id"], "test-123"); - assert_eq!(parsed["event"]["type"], "build_started"); + assert_eq!(parsed["build_started"]["invocation_id"], "test-123"); + assert_eq!(parsed["type"], "build_started"); } #[tokio::test] @@ -253,7 +251,6 @@ mod tests { let events = vec![ BuildEvent { - invocation_id: "test-1".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "test-1".to_string(), command: "build".to_string(), @@ -262,7 +259,6 @@ mod tests { })), }, BuildEvent { - invocation_id: "test-1".to_string(), event: Some(build_event::Event::TargetStarted(TargetStarted { target_id: "target-1".to_string(), label: "//foo:bar".to_string(), @@ -287,10 +283,10 @@ mod tests { assert_eq!(lines.len(), 2); let parsed0: serde_json::Value = serde_json::from_str(&lines[0]).unwrap(); - assert_eq!(parsed0["event"]["type"], "build_started"); + assert_eq!(parsed0["type"], "build_started"); let parsed1: serde_json::Value = serde_json::from_str(&lines[1]).unwrap(); - assert_eq!(parsed1["event"]["type"], "target_started"); + assert_eq!(parsed1["type"], "target_started"); } #[tokio::test] @@ -305,7 +301,6 @@ mod tests { let writer = JsonFileWriter::new(&file_path).unwrap(); let event = BuildEvent { - invocation_id: "new".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "new".to_string(), command: "build".to_string(), diff --git a/crates/build-events/src/subscribers/logger.rs b/crates/build-events/src/subscribers/logger.rs index 8d5ca518d3..c310123348 100644 --- a/crates/build-events/src/subscribers/logger.rs +++ b/crates/build-events/src/subscribers/logger.rs @@ -2,7 +2,7 @@ //! //! Logs build events using the tracing crate with structured fields. -use crate::events::{build_event, BuildEvent}; +use crate::events::{BuildEvent, build_event}; use crate::subscriber::{BuildEventSubscriber, SubscriberError}; use async_trait::async_trait; use tracing::info; @@ -146,7 +146,6 @@ mod tests { assert_eq!(subscriber.name(), "LoggerSubscriber"); let event = BuildEvent { - invocation_id: "test-123".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "test-123".to_string(), command: "build".to_string(), @@ -165,7 +164,6 @@ mod tests { let events = vec![ BuildEvent { - invocation_id: "test".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "test".to_string(), command: "build".to_string(), @@ -174,7 +172,6 @@ mod tests { })), }, BuildEvent { - invocation_id: "test".to_string(), event: Some(build_event::Event::TargetStarted(TargetStarted { target_id: "target-1".to_string(), label: "//foo:bar".to_string(), diff --git a/crates/build-events/tests/integration_test.rs b/crates/build-events/tests/integration_test.rs index 0000efd9ae..dc7855f302 100644 --- a/crates/build-events/tests/integration_test.rs +++ b/crates/build-events/tests/integration_test.rs @@ -6,8 +6,8 @@ use async_trait::async_trait; use build_events::{ BuildEvent, BuildEventBus, BuildEventDispatcher, BuildEventSubscriber, SubscriberError, events::{ - build_event, ActionCompleted, ActionStarted, BuildFinished, BuildStarted, TargetCompleted, - TargetKind, TargetStarted, current_millis, + ActionCompleted, ActionStarted, BuildFinished, BuildStarted, TargetCompleted, TargetKind, + TargetStarted, build_event, current_millis, }, subscribers::{JsonFileWriter, LoggerSubscriber}, }; @@ -48,7 +48,6 @@ async fn test_full_event_flow() { // Emit all event types bus.emit(BuildEvent { - invocation_id: "test-integration".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "test-integration".to_string(), command: "build".to_string(), @@ -58,7 +57,6 @@ async fn test_full_event_flow() { }); bus.emit(BuildEvent { - invocation_id: "test-integration".to_string(), event: Some(build_event::Event::TargetStarted(TargetStarted { target_id: "target-1".to_string(), label: "//foo:bar".to_string(), @@ -68,7 +66,6 @@ async fn test_full_event_flow() { }); bus.emit(BuildEvent { - invocation_id: "test-integration".to_string(), event: Some(build_event::Event::ActionStarted(ActionStarted { action_id: "action-1".to_string(), action_name: "CppCompile".to_string(), @@ -78,7 +75,6 @@ async fn test_full_event_flow() { }); bus.emit(BuildEvent { - invocation_id: "test-integration".to_string(), event: Some(build_event::Event::ActionCompleted(ActionCompleted { action_id: "action-1".to_string(), success: true, @@ -90,7 +86,6 @@ async fn test_full_event_flow() { }); bus.emit(BuildEvent { - invocation_id: "test-integration".to_string(), event: Some(build_event::Event::TargetCompleted(TargetCompleted { target_id: "target-1".to_string(), label: "//foo:bar".to_string(), @@ -103,9 +98,7 @@ async fn test_full_event_flow() { }); bus.emit(BuildEvent { - invocation_id: "test-integration".to_string(), event: Some(build_event::Event::BuildFinished(BuildFinished { - invocation_id: "test-integration".to_string(), success: true, timestamp_millis: current_millis(), error_message: None, @@ -145,7 +138,6 @@ async fn test_multiple_subscribers() { // Emit a few events for i in 0..3 { bus.emit(BuildEvent { - invocation_id: format!("test-{}", i), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: format!("test-{}", i), command: "build".to_string(), @@ -179,7 +171,6 @@ async fn test_late_subscriber_misses_early_events() { // Emit event before subscription bus.emit(BuildEvent { - invocation_id: "early".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "early".to_string(), command: "build".to_string(), @@ -201,7 +192,6 @@ async fn test_late_subscriber_misses_early_events() { // Emit event after subscription bus.emit(BuildEvent { - invocation_id: "late".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "late".to_string(), command: "build".to_string(), @@ -235,7 +225,6 @@ async fn test_bus_cloning() { // Emit from original bus bus.emit(BuildEvent { - invocation_id: "original".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "original".to_string(), command: "build".to_string(), @@ -246,7 +235,6 @@ async fn test_bus_cloning() { // Emit from cloned bus bus_clone.emit(BuildEvent { - invocation_id: "clone".to_string(), event: Some(build_event::Event::BuildStarted(BuildStarted { invocation_id: "clone".to_string(), command: "build".to_string(), diff --git a/crates/build-sandbox/src/executor.rs b/crates/build-sandbox/src/executor.rs index c1014e3b5a..f2d5766ea7 100644 --- a/crates/build-sandbox/src/executor.rs +++ b/crates/build-sandbox/src/executor.rs @@ -1,5 +1,5 @@ use build_events::events::{ - build_event, BuildEvent, FileCreated, TargetCompleted, TargetKind, TargetStarted, + BuildEvent, FileCreated, TargetCompleted, TargetKind, TargetStarted, build_event, }; use hakoniwa::Container; use std::fs; @@ -102,7 +102,7 @@ impl BuildExecutor { success, timestamp_millis: current_millis(), error_message, - outputs: vec![], // Outputs are collected later + outputs: vec![], // Outputs are collected later cache_hit: false, // Cache hit detection happens earlier })), }); diff --git a/crates/build-sandbox/src/executor_stub.rs b/crates/build-sandbox/src/executor_stub.rs index 61f92086a7..3f7721feab 100644 --- a/crates/build-sandbox/src/executor_stub.rs +++ b/crates/build-sandbox/src/executor_stub.rs @@ -13,11 +13,7 @@ impl BuildExecutor { todo!() } - pub async fn execute( - &self, - _: &BuildConfig, - _: &str, - ) -> Result { + pub async fn execute(&self, _: &BuildConfig, _: &str) -> Result { todo!() } @@ -33,11 +29,7 @@ impl BuildExecutor { todo!() } - async fn execute_in_container( - &self, - _: &BuildConfig, - _: &str, - ) -> Result<()> { + async fn execute_in_container(&self, _: &BuildConfig, _: &str) -> Result<()> { todo!() } diff --git a/crates/build-sandbox/src/lib.rs b/crates/build-sandbox/src/lib.rs index e58ffac73b..3000cfa649 100644 --- a/crates/build-sandbox/src/lib.rs +++ b/crates/build-sandbox/src/lib.rs @@ -40,8 +40,5 @@ pub async fn run_build( cache_dest_dir, )?; - Ok(BuildResult { - exit_code, - outputs, - }) + Ok(BuildResult { exit_code, outputs }) } diff --git a/crates/minimal/Cargo.toml b/crates/minimal/Cargo.toml index fb0d2c673f..3c543a543a 100644 --- a/crates/minimal/Cargo.toml +++ b/crates/minimal/Cargo.toml @@ -31,6 +31,7 @@ serde.workspace = true rayon.workspace = true tokio.workspace = true futures.workspace = true +uuid.workspace = true google-cloud-gax.workspace = true google-cloud-storage.workspace = true diff --git a/crates/minimal/src/cmd_build.rs b/crates/minimal/src/cmd_build.rs index 78a404c0d8..16ea873874 100644 --- a/crates/minimal/src/cmd_build.rs +++ b/crates/minimal/src/cmd_build.rs @@ -2,7 +2,7 @@ use crate::{Error, lockfile::PrebuiltsLock, remote_storage::RemoteStorage, run:: use crate::{GlobalArgs, PackagesArg}; use anyhow::Context; use build_events::events::{ - build_event, BuildEvent, BuildFinished, BuildMetadata, BuildStarted, current_millis, + BuildEvent, BuildFinished, BuildMetadata, BuildStarted, build_event, current_millis, }; use cache::{Cache, CacheBinProvider, LocalDir, RemoteBinProvider}; use graph::{DepGraph, ExecPlan, Transitives}; @@ -66,6 +66,7 @@ pub async fn cmd_build_impl( // Emit BuildStarted event using global event bus build_events::event_bus().emit(BuildEvent { event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: build_events::invocation_id().to_string(), command, timestamp_millis: current_millis(), working_directory, @@ -78,39 +79,45 @@ pub async fn cmd_build_impl( // Get git user if let Ok(output) = Command::new("git").args(["config", "user.name"]).output() && output.status.success() - && let Ok(user) = String::from_utf8(output.stdout) { - metadata.insert("user".to_string(), user.trim().to_string()); - } + && let Ok(user) = String::from_utf8(output.stdout) + { + metadata.insert("user".to_string(), user.trim().to_string()); + } // Get git branch if let Ok(output) = Command::new("git") .args(["rev-parse", "--abbrev-ref", "HEAD"]) .output() && output.status.success() - && let Ok(branch) = String::from_utf8(output.stdout) { - metadata.insert("branch".to_string(), branch.trim().to_string()); - } + && let Ok(branch) = String::from_utf8(output.stdout) + { + metadata.insert("branch".to_string(), branch.trim().to_string()); + } // Get git commit SHA if let Ok(output) = Command::new("git").args(["rev-parse", "HEAD"]).output() && output.status.success() - && let Ok(commit) = String::from_utf8(output.stdout) { - metadata.insert("commit".to_string(), commit.trim().to_string()); - } + && let Ok(commit) = String::from_utf8(output.stdout) + { + metadata.insert("commit".to_string(), commit.trim().to_string()); + } // Get git remote URL if let Ok(output) = Command::new("git") .args(["config", "remote.origin.url"]) .output() && output.status.success() - && let Ok(repo_url) = String::from_utf8(output.stdout) { - metadata.insert("repo_url".to_string(), repo_url.trim().to_string()); - } + && let Ok(repo_url) = String::from_utf8(output.stdout) + { + metadata.insert("repo_url".to_string(), repo_url.trim().to_string()); + } // Emit BuildMetadata event if we collected any metadata (using global event bus) if !metadata.is_empty() { build_events::event_bus().emit(BuildEvent { - event: Some(build_event::Event::BuildMetadata(BuildMetadata { metadata })), + event: Some(build_event::Event::BuildMetadata(BuildMetadata { + metadata, + })), }); } diff --git a/crates/minimal/src/main.rs b/crates/minimal/src/main.rs index 3a5e017ff9..c9048ee9c6 100644 --- a/crates/minimal/src/main.rs +++ b/crates/minimal/src/main.rs @@ -13,6 +13,7 @@ use tracing_indicatif::TickSettings; use tracing_indicatif::filter::IndicatifFilter; use tracing_indicatif::filter::hide_indicatif_span_fields; use tracing_subscriber::{EnvFilter, fmt, prelude::*}; +use uuid::Uuid; mod lockfile; mod paths; @@ -369,15 +370,20 @@ async fn main() -> Result<()> { let cli = Cli::parse(); cli.global_args.path_config().ensure_directories()?; - // Initialize global event bus and SpongeBob integration - let spongebob = build_events_proto::SpongeBob::new() - .await - .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; + // Generate invocation ID for this build + let invocation_id = Uuid::new_v4().to_string(); - // Initialize the global event bus + // Initialize global invocation ID and event bus + build_events::initialize_invocation_id(invocation_id.clone()); build_events::initialize_global_event_bus(); - let mut dispatcher = build_events::BuildEventDispatcher::new(build_events::event_bus().subscribe()); + // Initialize SpongeBob integration + let spongebob = build_events_proto::SpongeBob::new(invocation_id) + .await + .map_err(|e| anyhow::anyhow!("Failed to create SpongeBob client: {}", e))?; + + let mut dispatcher = + build_events::BuildEventDispatcher::new(build_events::event_bus().subscribe()); dispatcher.add_subscriber(Box::new(spongebob)); // Spawn dispatcher in background diff --git a/crates/minimal/src/run.rs b/crates/minimal/src/run.rs index 541fe400f9..57a3af7703 100644 --- a/crates/minimal/src/run.rs +++ b/crates/minimal/src/run.rs @@ -717,8 +717,11 @@ impl<'a> Run<'a> { s.spawn(move |_| { let _rt = tokio_runtime.enter(); - let result = - futures::executor::block_on(self2.do_build(&bsr, full_build, remote_cache)); + let result = futures::executor::block_on(self2.do_build( + &bsr, + full_build, + remote_cache, + )); match result { Err(e) => { From d67fd2419629ec7d258049d01fc8344245f297f4 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Wed, 15 Oct 2025 13:45:08 -0700 Subject: [PATCH 37/39] fix shutdown --- crates/build-events-proto/src/client.rs | 58 ++++++++++++++++++------ crates/build-events/src/bus.rs | 60 +++++++++++++++++++++---- crates/minimal/src/main.rs | 12 ++++- 3 files changed, 106 insertions(+), 24 deletions(-) diff --git a/crates/build-events-proto/src/client.rs b/crates/build-events-proto/src/client.rs index def0060e0a..d3f522c90a 100644 --- a/crates/build-events-proto/src/client.rs +++ b/crates/build-events-proto/src/client.rs @@ -3,6 +3,7 @@ use crate::proto::{BuildEvent, build_event_service_client::BuildEventServiceClient}; use async_trait::async_trait; use build_events::{BuildEventSubscriber, SubscriberError}; +use std::sync::Arc; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; @@ -25,9 +26,19 @@ pub type Result = std::result::Result; /// Creates a client streaming connection for publishing build events. /// All events are published through this stream. /// The server sends an Empty response when the stream closes. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct SpongeBob { - request_tx: mpsc::Sender, + request_tx: Arc>>>, + stream_handle: Arc>>>, +} + +impl Clone for SpongeBob { + fn clone(&self) -> Self { + Self { + request_tx: self.request_tx.clone(), + stream_handle: self.stream_handle.clone(), + } + } } impl SpongeBob { @@ -58,7 +69,7 @@ impl SpongeBob { // Spawn task to handle stream and wait for response let invocation_id_for_task = invocation_id.clone(); - tokio::spawn(async move { + let stream_handle = tokio::spawn(async move { let request_stream = ReceiverStream::new(request_rx); match client.publish_build_event_stream(request_stream).await { Ok(_response) => { @@ -78,7 +89,10 @@ impl SpongeBob { } }); - Ok(Self { request_tx }) + Ok(Self { + request_tx: Arc::new(tokio::sync::Mutex::new(Some(request_tx))), + stream_handle: Arc::new(tokio::sync::Mutex::new(Some(stream_handle))), + }) } /// Publish a build event to Spongebob via the streaming connection @@ -87,14 +101,17 @@ impl SpongeBob { /// The background task handles sending to the server. #[tracing::instrument(skip(self, event))] pub async fn publish_build_event(&self, event: BuildEvent) -> Result<()> { - self.request_tx - .send(event) - .await - .map_err(|e| SpongeBobError::Stream(format!("Failed to send event: {}", e)))?; - - tracing::debug!("Event queued for sending"); - - Ok(()) + let tx_lock = self.request_tx.lock().await; + if let Some(tx) = tx_lock.as_ref() { + tx.send(event) + .await + .map_err(|e| SpongeBobError::Stream(format!("Failed to send event: {}", e)))?; + + tracing::debug!("Event queued for sending"); + Ok(()) + } else { + Err(SpongeBobError::Stream("Stream already closed".to_string())) + } } /// Close the stream and wait for final response @@ -136,7 +153,22 @@ impl BuildEventSubscriber for SpongeBob { } async fn on_close(&self) -> std::result::Result<(), SubscriberError> { - // No cleanup needed - connection will be closed when dropped + tracing::debug!("Closing SpongeBob client and waiting for stream to complete"); + + // Drop the request sender to close the stream + { + let mut tx_lock = self.request_tx.lock().await; + tx_lock.take(); // This drops the sender, closing the channel + } + + // Wait for the stream handler task to complete and log the final URL + let mut handle_lock = self.stream_handle.lock().await; + if let Some(handle) = handle_lock.take() { + if let Err(e) = handle.await { + tracing::warn!("Stream handler task failed: {}", e); + } + } + Ok(()) } } diff --git a/crates/build-events/src/bus.rs b/crates/build-events/src/bus.rs index 8d1aedec98..1bc96b279e 100644 --- a/crates/build-events/src/bus.rs +++ b/crates/build-events/src/bus.rs @@ -4,7 +4,7 @@ //! broadcast channel to distribute events to multiple subscribers. use crate::events::BuildEvent; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use tokio::sync::broadcast; /// Central event bus for distributing build events to subscribers @@ -39,7 +39,7 @@ use tokio::sync::broadcast; /// ``` #[derive(Clone, Debug)] pub struct BuildEventBus { - sender: Arc>, + sender: Arc>>>, } impl BuildEventBus { @@ -60,7 +60,7 @@ impl BuildEventBus { pub fn new(capacity: usize) -> Self { let (sender, _) = broadcast::channel(capacity); Self { - sender: Arc::new(sender), + sender: Arc::new(Mutex::new(Some(sender))), } } @@ -88,10 +88,14 @@ impl BuildEventBus { /// }); /// ``` pub fn emit(&self, event: BuildEvent) { - // Use try_send to avoid blocking. If the channel is full, the event - // will be dropped. This is acceptable because we prioritize not - // blocking the build process over guaranteed delivery. - let _ = self.sender.send(event); + // Lock and send. If the sender is closed, drop the event. + // This is acceptable because we prioritize not blocking the build + // process over guaranteed delivery. + if let Ok(sender_lock) = self.sender.lock() { + if let Some(sender) = sender_lock.as_ref() { + let _ = sender.send(event); + } + } } /// Subscribe to build events @@ -110,7 +114,11 @@ impl BuildEventBus { /// // Both receivers will get events emitted from now on /// ``` pub fn subscribe(&self) -> broadcast::Receiver { - self.sender.subscribe() + let sender_lock = self.sender.lock().expect("Mutex poisoned"); + sender_lock + .as_ref() + .expect("Cannot subscribe to closed event bus") + .subscribe() } /// Get the current number of active subscribers @@ -125,7 +133,41 @@ impl BuildEventBus { /// assert_eq!(bus.receiver_count(), 1); /// ``` pub fn receiver_count(&self) -> usize { - self.sender.receiver_count() + if let Ok(sender_lock) = self.sender.lock() { + sender_lock + .as_ref() + .map(|s| s.receiver_count()) + .unwrap_or(0) + } else { + 0 + } + } + + /// Close the event bus, signaling all receivers to shut down + /// + /// This drops the sender, which causes all receivers to receive + /// `RecvError::Closed`, allowing them to shut down gracefully. + /// + /// # Example + /// ``` + /// use build_events::BuildEventBus; + /// + /// #[tokio::main] + /// async fn main() { + /// let bus = BuildEventBus::new(1000); + /// let mut receiver = bus.subscribe(); + /// + /// // Close the bus + /// bus.close(); + /// + /// // Receiver will get Closed error + /// assert!(receiver.recv().await.is_err()); + /// } + /// ``` + pub fn close(&self) { + if let Ok(mut sender_lock) = self.sender.lock() { + sender_lock.take(); // Drop the sender, closing the channel + } } } diff --git a/crates/minimal/src/main.rs b/crates/minimal/src/main.rs index c9048ee9c6..efcf3616ec 100644 --- a/crates/minimal/src/main.rs +++ b/crates/minimal/src/main.rs @@ -386,8 +386,8 @@ async fn main() -> Result<()> { build_events::BuildEventDispatcher::new(build_events::event_bus().subscribe()); dispatcher.add_subscriber(Box::new(spongebob)); - // Spawn dispatcher in background - tokio::spawn(async move { + // Spawn dispatcher in background and store the join handle + let dispatcher_handle = tokio::spawn(async move { dispatcher.run().await; }); @@ -403,6 +403,14 @@ async fn main() -> Result<()> { Command::Run(args) => cmd_run(args, &cli.global_args).await, }; + // Close the event bus to signal the dispatcher to shut down gracefully + build_events::event_bus().close(); + + // Wait for dispatcher to complete (which will wait for SpongeBob to log the final URL) + if let Err(e) = dispatcher_handle.await { + tracing::warn!("Dispatcher task failed: {}", e); + } + if let Err(e) = result { e.report_to_stderr(); std::process::exit(1); From 28b235fd030ac62c9be1095cb9e757f75680eeea Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Thu, 16 Oct 2025 09:53:13 -0700 Subject: [PATCH 38/39] Fix cargo-deny --- deny.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deny.toml b/deny.toml index 985ef8d8c2..77b4322acd 100644 --- a/deny.toml +++ b/deny.toml @@ -28,6 +28,7 @@ exceptions = [ [licenses.private] ignore = true +ignore-sources = ["https://buf.build/gen/cargo//"] [bans] skip = [ @@ -96,5 +97,8 @@ skip-tree = [ { name = "windows-targets", version = "=0.52.6" }, ] +[sources] +allow-registry = ["registry+https://github.com/rust-lang/crates.io-index", "sparse+https://buf.build/gen/cargo/"] + [sources.allow-org] github = ["souk4711"] From 7136d1ddfb21772ca242f4b59ed256f58bafcbd9 Mon Sep 17 00:00:00 2001 From: Brandon Weeks Date: Thu, 16 Oct 2025 11:59:13 -0700 Subject: [PATCH 39/39] replace json with protobuf text format --- Cargo.lock | 19 +- build_events.textproto | 21 ++ crates/build-events/Cargo.toml | 9 +- crates/build-events/src/lib.rs | 8 +- .../build-events/src/subscribers/json_file.rs | 321 ------------------ crates/build-events/src/subscribers/mod.rs | 6 +- .../build-events/src/subscribers/text_file.rs | 248 ++++++++++++++ crates/build-events/tests/integration_test.rs | 19 +- crates/minimal/src/main.rs | 11 + 9 files changed, 313 insertions(+), 349 deletions(-) create mode 100644 build_events.textproto delete mode 100644 crates/build-events/src/subscribers/json_file.rs create mode 100644 crates/build-events/src/subscribers/text_file.rs diff --git a/Cargo.lock b/Cargo.lock index c26e5a5127..bd431cba6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -347,8 +347,8 @@ version = "0.0.1" dependencies = [ "async-trait", "minimal_spongebob_community_neoeinstein-prost", - "serde", - "serde_json", + "prost 0.13.5", + "prost-reflect", "tempfile", "thiserror 2.0.17", "tokio", @@ -2815,7 +2815,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -2828,12 +2828,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn", ] +[[package]] +name = "prost-reflect" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37587d5a8a1b3dc9863403d084fc2254b91ab75a702207098837950767e2260b" +dependencies = [ + "logos", + "prost 0.13.5", + "prost-types 0.13.5", +] + [[package]] name = "prost-types" version = "0.13.5" diff --git a/build_events.textproto b/build_events.textproto new file mode 100644 index 0000000000..ee42e5e251 --- /dev/null +++ b/build_events.textproto @@ -0,0 +1,21 @@ +build_started { + invocation_id: "5dd2abb9-2cc7-4423-8995-685675c4c8ae" + timestamp_millis: 1760640889823 + working_directory: "/Users/bweeks/Code/minimal/minpkgs" + command: "--build-events-file build_events.textproto build --packages tar" +} +build_metadata { + metadata: [{ + key: "user" + value: "Brandon Weeks" + }, { + key: "commit" + value: "28b235fd030ac62c9be1095cb9e757f75680eeea" + }, { + key: "branch" + value: "spongebob2" + }, { + key: "repo_url" + value: "git@github.com:gominimal/minpkgs.git" + }] +} diff --git a/crates/build-events/Cargo.toml b/crates/build-events/Cargo.toml index 5477304f82..bb6cabf269 100644 --- a/crates/build-events/Cargo.toml +++ b/crates/build-events/Cargo.toml @@ -8,16 +8,13 @@ publish.workspace = true [dependencies] async-trait = "0.1" -serde_json.workspace = true -serde.workspace = true +minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251015195841-61cd123d44d0.1", registry = "buf" } +prost = "0.13" +prost-reflect = { version = "0.15", features = ["text-format"] } thiserror.workspace = true tokio.workspace = true tracing.workspace = true -# Proto types as single source of truth -minimal_spongebob_community_neoeinstein-prost = { version = "=0.4.0-20251015195841-61cd123d44d0.1", registry = "buf" } - [dev-dependencies] -serde_json.workspace = true tempfile.workspace = true tracing-subscriber.workspace = true diff --git a/crates/build-events/src/lib.rs b/crates/build-events/src/lib.rs index 3ff1256d0f..b8efc3dc4e 100644 --- a/crates/build-events/src/lib.rs +++ b/crates/build-events/src/lib.rs @@ -11,14 +11,14 @@ //! - **BuildEventBus** for publishing events using tokio broadcast channels //! - **BuildEventSubscriber** trait for consuming events //! - **BuildEventDispatcher** for managing multiple subscribers -//! - **Built-in subscribers** for logging and JSON file output +//! - **Built-in subscribers** for logging and protobuf text format file output //! //! # Architecture //! //! ```text //! Build Components → BuildEventBus → BuildEventDispatcher → Subscribers //! ├─ LoggerSubscriber -//! ├─ JsonFileWriter +//! ├─ TextFileWriter //! └─ Custom Subscribers //! ``` //! @@ -28,7 +28,7 @@ //! use build_events::{ //! BuildEventBus, BuildEventDispatcher, BuildEvent, //! events::{build_event, BuildStarted, current_millis}, -//! subscribers::{LoggerSubscriber, JsonFileWriter}, +//! subscribers::{LoggerSubscriber, TextFileWriter}, //! }; //! //! #[tokio::main] @@ -43,7 +43,7 @@ //! let mut dispatcher = BuildEventDispatcher::new(event_bus.subscribe()); //! dispatcher.add_subscriber(Box::new(LoggerSubscriber::new())); //! dispatcher.add_subscriber(Box::new( -//! JsonFileWriter::new("build_events.jsonl").unwrap() +//! TextFileWriter::new("build_events.textproto").unwrap() //! )); //! //! // Start event processing diff --git a/crates/build-events/src/subscribers/json_file.rs b/crates/build-events/src/subscribers/json_file.rs deleted file mode 100644 index 9cc579bfa3..0000000000 --- a/crates/build-events/src/subscribers/json_file.rs +++ /dev/null @@ -1,321 +0,0 @@ -//! JSON file writer subscriber implementation -//! -//! Writes build events to a file in JSON-lines format (one JSON object per line). - -use crate::events::{BuildEvent, TargetKind, build_event}; -use crate::subscriber::{BuildEventSubscriber, SubscriberError}; -use async_trait::async_trait; -use std::io::Write; -use std::path::{Path, PathBuf}; -use std::sync::Mutex; - -/// Subscriber that writes events to a JSON-lines file -/// -/// Each event is serialized to JSON and written as a single line to the file. -/// The file is automatically flushed after each write to ensure durability. -/// -/// # Example -/// ```no_run -/// use build_events::subscribers::JsonFileWriter; -/// -/// let writer = JsonFileWriter::new("build_events.jsonl").unwrap(); -/// ``` -pub struct JsonFileWriter { - file: Mutex, - path: PathBuf, -} - -impl JsonFileWriter { - /// Create a new JSON file writer - /// - /// Creates or truncates the file at the specified path. The file will - /// be written in JSON-lines format (one event per line). - /// - /// # Arguments - /// * `path` - Path to the output file - /// - /// # Errors - /// Returns an error if the file cannot be created or opened - /// - /// # Example - /// ```no_run - /// use build_events::subscribers::JsonFileWriter; - /// - /// let writer = JsonFileWriter::new("build_events.jsonl").unwrap(); - /// ``` - pub fn new>(path: P) -> std::io::Result { - let path = path.as_ref().to_path_buf(); - let file = std::fs::File::create(&path)?; - - Ok(Self { - file: Mutex::new(file), - path, - }) - } - - /// Get the path to the output file - pub fn path(&self) -> &Path { - &self.path - } -} - -impl std::fmt::Debug for JsonFileWriter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("JsonFileWriter") - .field("path", &self.path) - .finish() - } -} - -/// Convert proto BuildEvent to JSON value (custom serialization since proto types don't have serde) -fn event_to_json(event: &BuildEvent) -> serde_json::Value { - match &event.event { - Some(build_event::Event::BuildStarted(e)) => { - serde_json::json!({ - "type": "build_started", - "build_started": { - "invocation_id": e.invocation_id, - "command": e.command, - "timestamp_millis": e.timestamp_millis, - "working_directory": e.working_directory, - } - }) - } - Some(build_event::Event::BuildFinished(e)) => { - serde_json::json!({ - "type": "build_finished", - "build_finished": { - "success": e.success, - "timestamp_millis": e.timestamp_millis, - "error_message": e.error_message, - } - }) - } - Some(build_event::Event::TargetStarted(e)) => { - let target_kind = match TargetKind::try_from(e.target_kind) { - Ok(TargetKind::Binary) => "binary", - Ok(TargetKind::Library) => "library", - Ok(TargetKind::Test) => "test", - _ => "unspecified", - }; - serde_json::json!({ - "type": "target_started", - "target_started": { - "target_id": e.target_id, - "label": e.label, - "target_kind": target_kind, - "timestamp_millis": e.timestamp_millis, - } - }) - } - Some(build_event::Event::TargetCompleted(e)) => { - serde_json::json!({ - "type": "target_completed", - "target_completed": { - "target_id": e.target_id, - "label": e.label, - "success": e.success, - "timestamp_millis": e.timestamp_millis, - "error_message": e.error_message, - "outputs": e.outputs, - "cache_hit": e.cache_hit, - } - }) - } - Some(build_event::Event::ActionStarted(e)) => { - serde_json::json!({ - "type": "action_started", - "action_started": { - "action_id": e.action_id, - "action_name": e.action_name, - "target_id": e.target_id, - "timestamp_millis": e.timestamp_millis, - } - }) - } - Some(build_event::Event::ActionCompleted(e)) => { - serde_json::json!({ - "type": "action_completed", - "action_completed": { - "action_id": e.action_id, - "success": e.success, - "timestamp_millis": e.timestamp_millis, - "exit_code": e.exit_code, - "stdout": e.stdout, - "stderr": e.stderr, - } - }) - } - Some(build_event::Event::BuildMetadata(e)) => { - serde_json::json!({ - "type": "build_metadata", - "build_metadata": { - "metadata": e.metadata, - } - }) - } - Some(build_event::Event::FileCreated(e)) => { - serde_json::json!({ - "type": "file_created", - "file_created": { - "target_id": e.target_id, - "name": e.name, - "size": e.contents.len(), - "timestamp_millis": e.timestamp_millis, - } - }) - } - None => { - serde_json::json!({ - "type": "unknown", - }) - } - } -} - -#[async_trait] -impl BuildEventSubscriber for JsonFileWriter { - async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { - let json = event_to_json(event); - let json_string = serde_json::to_string(&json).map_err(|e| { - SubscriberError::Serialization(format!("Failed to serialize event: {}", e)) - })?; - - let mut file = self.file.lock().unwrap(); - writeln!(file, "{}", json_string)?; - file.flush()?; - - Ok(()) - } - - fn name(&self) -> &str { - "JsonFileWriter" - } - - async fn on_close(&self) -> Result<(), SubscriberError> { - let mut file = self.file.lock().unwrap(); - file.flush()?; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::events::{BuildStarted, TargetStarted, current_millis}; - use std::io::{BufRead, BufReader}; - - #[tokio::test] - async fn test_json_file_writer() { - let temp_dir = tempfile::tempdir().unwrap(); - let file_path = temp_dir.path().join("events.jsonl"); - - let writer = JsonFileWriter::new(&file_path).unwrap(); - assert_eq!(writer.name(), "JsonFileWriter"); - assert_eq!(writer.path(), file_path.as_path()); - - let event = BuildEvent { - event: Some(build_event::Event::BuildStarted(BuildStarted { - invocation_id: "test-123".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - })), - }; - - let result = writer.on_event(&event).await; - assert!(result.is_ok()); - - // Close and flush - writer.on_close().await.unwrap(); - drop(writer); - - // Read back and verify - let file = std::fs::File::open(&file_path).unwrap(); - let reader = BufReader::new(file); - let lines: Vec = reader.lines().map(|l| l.unwrap()).collect(); - - assert_eq!(lines.len(), 1); - - let parsed: serde_json::Value = serde_json::from_str(&lines[0]).unwrap(); - assert_eq!(parsed["build_started"]["invocation_id"], "test-123"); - assert_eq!(parsed["type"], "build_started"); - } - - #[tokio::test] - async fn test_json_file_multiple_events() { - let temp_dir = tempfile::tempdir().unwrap(); - let file_path = temp_dir.path().join("events.jsonl"); - - let writer = JsonFileWriter::new(&file_path).unwrap(); - - let events = vec![ - BuildEvent { - event: Some(build_event::Event::BuildStarted(BuildStarted { - invocation_id: "test-1".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - })), - }, - BuildEvent { - event: Some(build_event::Event::TargetStarted(TargetStarted { - target_id: "target-1".to_string(), - label: "//foo:bar".to_string(), - target_kind: TargetKind::Binary as i32, - timestamp_millis: current_millis(), - })), - }, - ]; - - for event in &events { - writer.on_event(event).await.unwrap(); - } - - writer.on_close().await.unwrap(); - drop(writer); - - // Read back and verify - let file = std::fs::File::open(&file_path).unwrap(); - let reader = BufReader::new(file); - let lines: Vec = reader.lines().map(|l| l.unwrap()).collect(); - - assert_eq!(lines.len(), 2); - - let parsed0: serde_json::Value = serde_json::from_str(&lines[0]).unwrap(); - assert_eq!(parsed0["type"], "build_started"); - - let parsed1: serde_json::Value = serde_json::from_str(&lines[1]).unwrap(); - assert_eq!(parsed1["type"], "target_started"); - } - - #[tokio::test] - async fn test_json_file_truncates_existing() { - let temp_dir = tempfile::tempdir().unwrap(); - let file_path = temp_dir.path().join("events.jsonl"); - - // Write some initial data - std::fs::write(&file_path, "old data\n").unwrap(); - - // Create writer (should truncate) - let writer = JsonFileWriter::new(&file_path).unwrap(); - - let event = BuildEvent { - event: Some(build_event::Event::BuildStarted(BuildStarted { - invocation_id: "new".to_string(), - command: "build".to_string(), - timestamp_millis: current_millis(), - working_directory: "/tmp".to_string(), - })), - }; - - writer.on_event(&event).await.unwrap(); - writer.on_close().await.unwrap(); - drop(writer); - - // Read back - should only have new data - let content = std::fs::read_to_string(&file_path).unwrap(); - assert!(!content.contains("old data")); - assert!(content.contains("new")); - } -} diff --git a/crates/build-events/src/subscribers/mod.rs b/crates/build-events/src/subscribers/mod.rs index a84da1d825..cb635256f6 100644 --- a/crates/build-events/src/subscribers/mod.rs +++ b/crates/build-events/src/subscribers/mod.rs @@ -2,10 +2,10 @@ //! //! This module provides ready-to-use subscriber implementations: //! - [`LoggerSubscriber`]: Logs events using tracing -//! - [`JsonFileWriter`]: Writes events to a JSON-lines file +//! - [`TextFileWriter`]: Writes events to a protobuf text format file -mod json_file; mod logger; +mod text_file; -pub use json_file::JsonFileWriter; pub use logger::LoggerSubscriber; +pub use text_file::TextFileWriter; diff --git a/crates/build-events/src/subscribers/text_file.rs b/crates/build-events/src/subscribers/text_file.rs new file mode 100644 index 0000000000..4f85b03859 --- /dev/null +++ b/crates/build-events/src/subscribers/text_file.rs @@ -0,0 +1,248 @@ +//! Protobuf text format file writer subscriber implementation +//! +//! Writes build events to a file in protobuf text format (one message per separator). + +use crate::events::BuildEvent; +use crate::subscriber::{BuildEventSubscriber, SubscriberError}; +use async_trait::async_trait; +use prost::Message; +use prost_reflect::{DescriptorPool, DynamicMessage, text_format::FormatOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::{LazyLock, Mutex}; + +/// Global descriptor pool for BuildEvent messages +static DESCRIPTOR_POOL: LazyLock = LazyLock::new(|| { + // Start with global pool that includes well-known types + let mut pool = DescriptorPool::global(); + + // Decode and add our custom file descriptors + let fds = prost_reflect::prost_types::FileDescriptorSet::decode( + minimal_spongebob_community_neoeinstein_prost::spongebob::v1::FILE_DESCRIPTOR_SET, + ) + .expect("Failed to decode file descriptor set"); + + // Add all file descriptors to the pool + for file in fds.file { + pool.add_file_descriptor_proto(file) + .expect("Failed to add file descriptor to pool"); + } + + pool +}); + +/// Subscriber that writes events to a protobuf text format file +/// +/// Each event is serialized to protobuf text format and written to the file. +/// Messages are separated by a line of dashes for readability. +/// The file is automatically flushed after each write to ensure durability. +/// +/// # Example +/// ```no_run +/// use build_events::subscribers::TextFileWriter; +/// +/// let writer = TextFileWriter::new("build_events.textproto").unwrap(); +/// ``` +pub struct TextFileWriter { + file: Mutex, + path: PathBuf, +} + +impl TextFileWriter { + /// Create a new protobuf text format file writer + /// + /// Creates or truncates the file at the specified path. The file will + /// be written in protobuf text format (one event per separator). + /// + /// # Arguments + /// * `path` - Path to the output file + /// + /// # Errors + /// Returns an error if the file cannot be created or opened + /// + /// # Example + /// ```no_run + /// use build_events::subscribers::TextFileWriter; + /// + /// let writer = TextFileWriter::new("build_events.textproto").unwrap(); + /// ``` + pub fn new>(path: P) -> std::io::Result { + let path = path.as_ref().to_path_buf(); + let file = std::fs::File::create(&path)?; + + Ok(Self { + file: Mutex::new(file), + path, + }) + } + + /// Get the path to the output file + pub fn path(&self) -> &Path { + &self.path + } +} + +impl std::fmt::Debug for TextFileWriter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TextFileWriter") + .field("path", &self.path) + .finish() + } +} + +/// Convert proto BuildEvent to text format string +fn event_to_text_format(event: &BuildEvent) -> Result { + let descriptor = DESCRIPTOR_POOL + .get_message_by_name("spongebob.v1.BuildEvent") + .ok_or_else(|| { + SubscriberError::Serialization("Failed to get BuildEvent descriptor".to_string()) + })?; + + let bytes = event.encode_to_vec(); + + let dynamic_message = DynamicMessage::decode(descriptor, bytes.as_slice()).map_err(|e| { + SubscriberError::Serialization(format!("Failed to decode BuildEvent: {}", e)) + })?; + + let options = FormatOptions::new().pretty(true); + Ok(dynamic_message.to_text_format_with_options(&options)) +} + +#[async_trait] +impl BuildEventSubscriber for TextFileWriter { + async fn on_event(&self, event: &BuildEvent) -> Result<(), SubscriberError> { + let mut file = self.file.lock().unwrap(); + writeln!(file, "{}", event_to_text_format(event)?)?; + + Ok(()) + } + + fn name(&self) -> &str { + "TextFileWriter" + } + + async fn on_close(&self) -> Result<(), SubscriberError> { + let mut file = self.file.lock().unwrap(); + file.flush()?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::{BuildStarted, TargetKind, TargetStarted, build_event, current_millis}; + + #[tokio::test] + async fn test_text_file_writer() { + let temp_dir = tempfile::tempdir().unwrap(); + let file_path = temp_dir.path().join("events.textproto"); + + let writer = TextFileWriter::new(&file_path).unwrap(); + assert_eq!(writer.name(), "TextFileWriter"); + assert_eq!(writer.path(), file_path.as_path()); + + let event = BuildEvent { + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test-123".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }; + + let result = writer.on_event(&event).await; + assert!(result.is_ok()); + + // Close and flush + writer.on_close().await.unwrap(); + drop(writer); + + // Read back and verify it contains expected fields + let content = std::fs::read_to_string(&file_path).unwrap(); + assert!(content.contains("build_started")); + assert!(content.contains("invocation_id")); + assert!(content.contains("test-123")); + assert!(content.contains("command")); + assert!(content.contains("build")); + } + + #[tokio::test] + async fn test_text_file_multiple_events() { + let temp_dir = tempfile::tempdir().unwrap(); + let file_path = temp_dir.path().join("events.textproto"); + + let writer = TextFileWriter::new(&file_path).unwrap(); + + let events = vec![ + BuildEvent { + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "test-1".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }, + BuildEvent { + event: Some(build_event::Event::TargetStarted(TargetStarted { + target_id: "target-1".to_string(), + label: "//foo:bar".to_string(), + target_kind: TargetKind::Binary as i32, + timestamp_millis: current_millis(), + })), + }, + ]; + + for event in &events { + writer.on_event(event).await.unwrap(); + } + + writer.on_close().await.unwrap(); + drop(writer); + + // Read back and verify both events are present + let content = std::fs::read_to_string(&file_path).unwrap(); + assert!(content.contains("build_started")); + assert!(content.contains("test-1")); + assert!(content.contains("target_started")); + assert!(content.contains("target-1")); + assert!(content.contains("//foo:bar")); + } + + #[tokio::test] + async fn test_text_file_truncates_existing() { + let temp_dir = tempfile::tempdir().unwrap(); + let file_path = temp_dir.path().join("events.textproto"); + + // Write some initial data + std::fs::write(&file_path, "old data\n").unwrap(); + + // Create writer (should truncate) + let writer = TextFileWriter::new(&file_path).unwrap(); + + let event = BuildEvent { + event: Some(build_event::Event::BuildStarted(BuildStarted { + invocation_id: "new".to_string(), + command: "build".to_string(), + timestamp_millis: current_millis(), + working_directory: "/tmp".to_string(), + })), + }; + + writer.on_event(&event).await.unwrap(); + writer.on_close().await.unwrap(); + drop(writer); + + // Read back - should only have new data + let content = std::fs::read_to_string(&file_path).unwrap(); + assert!(!content.contains("old data")); + assert!(content.contains("new")); + } + + #[test] + fn test_descriptor_pool_initialization() { + // Verify the descriptor pool can be initialized and contains BuildEvent + let descriptor = DESCRIPTOR_POOL.get_message_by_name("spongebob.v1.BuildEvent"); + assert!(descriptor.is_some()); + } +} diff --git a/crates/build-events/tests/integration_test.rs b/crates/build-events/tests/integration_test.rs index dc7855f302..f3f892f0ed 100644 --- a/crates/build-events/tests/integration_test.rs +++ b/crates/build-events/tests/integration_test.rs @@ -9,7 +9,7 @@ use build_events::{ ActionCompleted, ActionStarted, BuildFinished, BuildStarted, TargetCompleted, TargetKind, TargetStarted, build_event, current_millis, }, - subscribers::{JsonFileWriter, LoggerSubscriber}, + subscribers::{LoggerSubscriber, TextFileWriter}, }; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -119,14 +119,14 @@ async fn test_full_event_flow() { #[tokio::test] async fn test_multiple_subscribers() { let temp_dir = tempfile::tempdir().unwrap(); - let json_path = temp_dir.path().join("events.jsonl"); + let text_path = temp_dir.path().join("events.textproto"); let bus = BuildEventBus::new(100); let count = Arc::new(AtomicUsize::new(0)); let mut dispatcher = BuildEventDispatcher::new(bus.subscribe()); dispatcher.add_subscriber(Box::new(LoggerSubscriber::new())); - dispatcher.add_subscriber(Box::new(JsonFileWriter::new(&json_path).unwrap())); + dispatcher.add_subscriber(Box::new(TextFileWriter::new(&text_path).unwrap())); dispatcher.add_subscriber(Box::new(CountingSubscriber { count: count.clone(), })); @@ -154,15 +154,12 @@ async fn test_multiple_subscribers() { // Verify counter subscriber received events assert_eq!(count.load(Ordering::SeqCst), 3); - // Verify JSON file has events - let content = std::fs::read_to_string(&json_path).unwrap(); - let lines: Vec<&str> = content.lines().collect(); - assert_eq!(lines.len(), 3); + // Verify text format file has events + let content = std::fs::read_to_string(&text_path).unwrap(); - // Verify each line is valid JSON (we can parse as serde_json::Value even though BuildEvent doesn't have serde) - for line in lines { - let _: serde_json::Value = serde_json::from_str(line).unwrap(); - } + // Each event should have "build_started" in the output + let event_count = content.matches("build_started").count(); + assert_eq!(event_count, 3); } #[tokio::test] diff --git a/crates/minimal/src/main.rs b/crates/minimal/src/main.rs index efcf3616ec..a1c6f3a1e3 100644 --- a/crates/minimal/src/main.rs +++ b/crates/minimal/src/main.rs @@ -109,6 +109,10 @@ pub struct GlobalArgs { /// Configure the number of parallel builds #[arg(short, long, default_value_t = default_parallelism())] num_parallel_builds: usize, + + /// Write build events to a protobuf text format file + #[arg(long)] + build_events_file: Option, } fn default_parallelism() -> usize { @@ -386,6 +390,13 @@ async fn main() -> Result<()> { build_events::BuildEventDispatcher::new(build_events::event_bus().subscribe()); dispatcher.add_subscriber(Box::new(spongebob)); + // Add text file writer if requested + if let Some(events_file) = &cli.global_args.build_events_file { + let writer = build_events::subscribers::TextFileWriter::new(events_file) + .map_err(|e| anyhow::anyhow!("Failed to create events file writer: {}", e))?; + dispatcher.add_subscriber(Box::new(writer)); + } + // Spawn dispatcher in background and store the join handle let dispatcher_handle = tokio::spawn(async move { dispatcher.run().await;