diff --git a/Cargo.lock b/Cargo.lock index e7ba1010fc..1b0b266be5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3999,6 +3999,7 @@ version = "0.0.1" dependencies = [ "anyhow", "bytes", + "check", "common", "futures", "graph", diff --git a/crates/check/src/harness.rs b/crates/check/src/harness.rs index e95565743e..5b65242d60 100644 --- a/crates/check/src/harness.rs +++ b/crates/check/src/harness.rs @@ -130,7 +130,7 @@ fn check_harness_packages_valid( program: &mut Program, ) -> Result { let mut out = CheckResult { - check: "packages valid", + check: "packages valid".into(), verdict: CheckVerdict::Skip, err: vec![], }; diff --git a/crates/check/src/lib.rs b/crates/check/src/lib.rs index 1f50fe7160..4a72ec4488 100644 --- a/crates/check/src/lib.rs +++ b/crates/check/src/lib.rs @@ -12,6 +12,7 @@ use lcache::{Cache, CacheErr, LocalDir}; use op::{Options, Runnable, StandaloneTest}; use ot::{OpTracker, Operation}; use regex::Regex; +use std::borrow::Cow; use std::cmp::Ordering; use std::fmt; use std::fmt::Display; @@ -123,7 +124,7 @@ impl Display for CheckVerdict { #[derive(Debug, Clone)] pub struct CheckResult { - pub check: &'static str, + pub check: Cow<'static, str>, pub verdict: CheckVerdict, pub err: Vec, } @@ -131,7 +132,7 @@ pub struct CheckResult { impl CheckResult { pub(crate) fn parse_failure(msg: String) -> Self { CheckResult { - check: "parse", + check: "parse".into(), verdict: CheckVerdict::Fail, err: vec![msg], } @@ -139,7 +140,7 @@ impl CheckResult { pub(crate) fn profile_name_skip() -> Self { CheckResult { - check: "profile name matches dir", + check: "profile name matches dir".into(), verdict: CheckVerdict::Skip, err: vec![], } @@ -147,7 +148,7 @@ impl CheckResult { pub(crate) fn profile_name_pass() -> Self { CheckResult { - check: "profile name matches dir", + check: "profile name matches dir".into(), verdict: CheckVerdict::Pass, err: vec![], } @@ -155,7 +156,7 @@ impl CheckResult { pub(crate) fn profile_name_fail(msg: String) -> Self { CheckResult { - check: "profile name matches dir", + check: "profile name matches dir".into(), verdict: CheckVerdict::Fail, err: vec![msg], } @@ -163,21 +164,21 @@ impl CheckResult { pub(crate) fn harness_name_skip() -> Self { CheckResult { - check: "harness name matches dir", + check: "harness name matches dir".into(), verdict: CheckVerdict::Skip, err: vec![], } } pub(crate) fn harness_name_pass() -> Self { CheckResult { - check: "harness name matches dir", + check: "harness name matches dir".into(), verdict: CheckVerdict::Pass, err: vec![], } } pub(crate) fn harness_name_fail(msg: String) -> Self { CheckResult { - check: "harness name matches dir", + check: "harness name matches dir".into(), verdict: CheckVerdict::Fail, err: vec![msg], } @@ -185,21 +186,21 @@ impl CheckResult { pub(crate) fn harness_regexes_skip() -> Self { CheckResult { - check: "project_matchers regexes", + check: "project_matchers regexes".into(), verdict: CheckVerdict::Skip, err: vec![], } } pub(crate) fn harness_regexes_pass() -> Self { CheckResult { - check: "project_matchers regexes", + check: "project_matchers regexes".into(), verdict: CheckVerdict::Pass, err: vec![], } } pub(crate) fn harness_regexes_fail(msg: String) -> Self { CheckResult { - check: "project_matchers regexes", + check: "project_matchers regexes".into(), verdict: CheckVerdict::Fail, err: vec![msg], } @@ -207,21 +208,21 @@ impl CheckResult { pub(crate) fn harness_predicates_skip() -> Self { CheckResult { - check: "project_matchers predicates", + check: "project_matchers predicates".into(), verdict: CheckVerdict::Skip, err: vec![], } } pub(crate) fn harness_predicates_pass() -> Self { CheckResult { - check: "project_matchers predicates", + check: "project_matchers predicates".into(), verdict: CheckVerdict::Pass, err: vec![], } } pub(crate) fn harness_predicates_fail(msg: String) -> Self { CheckResult { - check: "project_matchers predicates", + check: "project_matchers predicates".into(), verdict: CheckVerdict::Fail, err: vec![msg], } @@ -545,7 +546,7 @@ impl FileBasedChecker for ParseCheck { ) -> Result { if ctx.skip_checkers.contains(&"parse".to_string()) { return Ok(CheckResult { - check: "parse", + check: "parse".into(), verdict: CheckVerdict::Skip, err: vec![], }); @@ -568,7 +569,7 @@ impl FileBasedChecker for ParseCheck { Ok(p) => p, Err(e) => { return Ok(CheckResult { - check: "parse", + check: "parse".into(), verdict: CheckVerdict::Fail, err: vec![format!("{}", e)], }); @@ -585,7 +586,7 @@ impl FileBasedChecker for ParseCheck { if let Err(e) = program.typecheck(nickel_lang_core::typecheck::TypecheckMode::Walk) { return Ok(CheckResult { - check: "parse", + check: "parse".into(), verdict: CheckVerdict::Fail, err: vec![report_as_str( &mut program.files(), @@ -596,7 +597,7 @@ impl FileBasedChecker for ParseCheck { } if let Err(e) = program.compile() { return Ok(CheckResult { - check: "parse", + check: "parse".into(), verdict: CheckVerdict::Fail, err: vec![report_as_str( &mut program.files(), @@ -608,7 +609,7 @@ impl FileBasedChecker for ParseCheck { drop(generated_lib_dir); Ok(CheckResult { - check: "parse", + check: "parse".into(), verdict: CheckVerdict::Pass, err: vec![], }) @@ -633,7 +634,7 @@ impl FileBasedChecker for ImportLineCheck { let fix = ctx.fix; let mut result = CheckResult { verdict: CheckVerdict::Pass, - check: "import line", + check: "import line".into(), err: vec![], }; if ctx.skip_checkers.contains(&"import line".to_string()) { @@ -747,7 +748,7 @@ impl FileBasedChecker for FmtCheck { let fix = ctx.fix; let mut result = CheckResult { verdict: CheckVerdict::Skip, - check: "fmt", + check: "fmt".into(), err: vec![], }; if ctx.skip_checkers.contains(&"fmt".to_string()) { @@ -814,7 +815,7 @@ impl GraphBasedChecker for StandaloneTestCheck { let cache = ctx.cache.clone(); let mut result = CheckResult { verdict: CheckVerdict::Skip, - check: "standalone tests", + check: "standalone tests".into(), err: vec![], }; if ctx.skip_checkers.contains(&"standalone tests".to_string()) { @@ -921,7 +922,7 @@ impl FileBasedChecker for ImportsCheck { ) -> Result { let mut result = CheckResult { verdict: CheckVerdict::Pass, - check: "imports", + check: "imports".into(), err: vec![], }; if ctx.skip_checkers.contains(&"imports".to_string()) { @@ -1029,7 +1030,7 @@ impl GraphBasedChecker for BuildScriptDisallowedPatterns { ) -> Result { let mut result = CheckResult { verdict: CheckVerdict::Skip, - check: "build script disallowed-patterns", + check: "build script disallowed-patterns".into(), err: vec![], }; if ctx @@ -1105,7 +1106,7 @@ impl GraphBasedChecker for BuildScriptIsExecutable { ) -> Result { let mut result = CheckResult { verdict: CheckVerdict::Skip, - check: "build scripts are executable", + check: "build scripts are executable".into(), err: vec![], }; if ctx diff --git a/crates/check/src/naming.rs b/crates/check/src/naming.rs index acae52955b..322e1f7dad 100644 --- a/crates/check/src/naming.rs +++ b/crates/check/src/naming.rs @@ -19,7 +19,7 @@ impl crate::GraphBasedChecker for SpecNameMatchesDir { ) -> Result { let mut result = CheckResult { verdict: CheckVerdict::Skip, - check: "spec name matches dir", + check: "spec name matches dir".into(), err: vec![], }; if ctx @@ -59,7 +59,7 @@ impl crate::GraphBasedChecker for SpecNameValid { ) -> Result { let mut result = CheckResult { verdict: CheckVerdict::Skip, - check: "spec name valid", + check: "spec name valid".into(), err: vec![], }; if ctx.skip_checkers.contains(&"spec name valid".to_string()) { @@ -97,7 +97,7 @@ impl crate::GraphBasedChecker for CycleBreakerNaming { ) -> Result { let mut result = CheckResult { verdict: CheckVerdict::Skip, - check: "cycle breaker naming", + check: "cycle breaker naming".into(), err: vec![], }; if ctx @@ -147,7 +147,7 @@ impl crate::GraphBasedChecker for OutputNaming { ) -> Result { let mut result = CheckResult { verdict: CheckVerdict::Skip, - check: "output naming", + check: "output naming".into(), err: vec![], }; if ctx.skip_checkers.contains(&"output naming".to_string()) { @@ -212,7 +212,7 @@ impl crate::GraphBasedChecker for EnumerateBins { let cache = ctx.cache.clone(); let mut result = CheckResult { verdict: CheckVerdict::Skip, - check: "enumerate bins", + check: "enumerate bins".into(), err: vec![], }; if ctx.skip_checkers.contains(&"enumerate bins".to_string()) { diff --git a/crates/check/src/outputs.rs b/crates/check/src/outputs.rs index a8f38fef0a..bf96b30b3a 100644 --- a/crates/check/src/outputs.rs +++ b/crates/check/src/outputs.rs @@ -26,7 +26,7 @@ impl crate::GraphBasedChecker for OutputTypesValid { let cache = ctx.cache.clone(); let mut result = CheckResult { verdict: CheckVerdict::Skip, - check: "output types valid", + check: "output types valid".into(), err: vec![], }; if ctx @@ -123,7 +123,7 @@ impl crate::GraphBasedChecker for MissingRuntimeDeps { let cache = ctx.cache.clone(); let mut result = CheckResult { verdict: CheckVerdict::Skip, - check: "missing runtime_deps", + check: "missing runtime_deps".into(), err: vec![], }; if ctx diff --git a/crates/check/src/profile.rs b/crates/check/src/profile.rs index 9a6f655cc1..ff73864a7b 100644 --- a/crates/check/src/profile.rs +++ b/crates/check/src/profile.rs @@ -86,7 +86,7 @@ fn check_profile_parses( program: &mut Program, ) -> Result { let mut out = CheckResult { - check: "profile parses", + check: "profile parses".into(), err: vec![], verdict: CheckVerdict::Pass, }; diff --git a/crates/remote-client/Cargo.toml b/crates/remote-client/Cargo.toml index f4b7458d1a..d54916a6ff 100644 --- a/crates/remote-client/Cargo.toml +++ b/crates/remote-client/Cargo.toml @@ -18,6 +18,7 @@ tokio-stream.workspace = true tokio-util.workspace = true tonic.workspace = true +check.workspace = true common.workspace = true graph.workspace = true orchestrator.workspace = true diff --git a/crates/remote-client/src/lib.rs b/crates/remote-client/src/lib.rs index 9801e78e85..fd105eb896 100644 --- a/crates/remote-client/src/lib.rs +++ b/crates/remote-client/src/lib.rs @@ -16,6 +16,7 @@ use tonic::transport::{Channel, Error as TransportError}; type CreateEnvStream = Pin + Send>>; type CreateBuildStream = Pin + Send>>; +type CheckStream = Pin + Send>>; /// Errors that can occur when driving a remote client. #[derive(Debug)] @@ -424,6 +425,159 @@ where Ok(tempfile) } + + pub async fn check( + &mut self, + dir: &Path, + args: &CheckArgs, + ) -> Result< + mpsc::UnboundedReceiver<(check::CheckObj, Result, Error>)>, + Error, + > { + use check_message::Msg; + let client_id = format!( + "{}-{}", + common::random_alphanumeric(8), + common::random_alphanumeric(8) + ); + + // Build a tarball of the directory to stream as file chunks. + use stream_config::*; + let dir = dir.to_path_buf(); + let file = tokio::task::spawn_blocking(move || -> Result { + let (mut file, _hash) = common::archive::compress_dir(&dir, Some(-5), &None) + .map_err(|e| Error::Other(e.into()))?; + std::io::Seek::rewind(&mut file).map_err(|e| Error::Other(e.into()))?; + Ok(file) + }) + .await + .map_err(|e| Error::Other(e.into())) + .and_then(|r| r)?; + + let chunk_stream: CheckStream = Box::pin( + tokio_util::io::ReaderStream::new(tokio::fs::File::from_std(file)) + .filter_map(|result| async { result.ok() }) + .map(|bytes| CheckMessage { + msg: Some(Msg::Chunk(bytes.to_vec())), + }), + ); + + let request_msg = tokio_stream::once(CheckMessage { + msg: Some(Msg::Request(CheckRequest { + client_id, + files_stream: Some(StreamConfig { + format: Some(Format::Files(FilesFormat { + tarball: Some(TarballFormat { + compression: TarballCompression::TarZst.into(), + }), + })), + kind: StreamKind::SkWorktree.into(), + }), + filter_names: args.filter_names.clone(), + skip_checkers: args.skip_checkers.clone(), + packages: args.packages, + profiles: args.profiles, + harnesses: args.harnesses, + remote_cache_gcs_bucket: args.remote_cache_gcs_bucket.clone(), + })), + }); + + let mut rpc = self + .c + .check(request_msg.chain(chunk_stream)) + .await + .map_err(|e| Error::Other(e.into()))? + .into_inner(); + + let (tx, rx) = mpsc::unbounded(); + tokio::spawn(async move { + while let Some(msg) = rpc.next().await { + let msg = match msg { + Ok(m) => m, + Err(e) => { + let _ = tx.unbounded_send(( + check::CheckObj::Package(String::new()), + Err(Error::Other(e.into())), + )); + break; + } + }; + match msg.msg { + Some(check_response::Msg::Error(e)) => { + let _ = tx.unbounded_send(( + check::CheckObj::Package(String::new()), + Err(Error::Other(anyhow!("check failed: {}", e.msg).into())), + )); + break; + } + Some(check_response::Msg::Check(c)) => { + let obj = match c.obj.and_then(|o| o.msg) { + Some(check_object::Msg::Package(n)) => check::CheckObj::Package(n), + Some(check_object::Msg::Profile(n)) => check::CheckObj::Profile(n), + Some(check_object::Msg::Harness(n)) => check::CheckObj::Harness(n), + None => { + let _ = tx.unbounded_send(( + check::CheckObj::Package(String::new()), + Err(Error::Other( + anyhow!("check response missing object").into(), + )), + )); + break; + } + }; + let check_results: Result, Error> = c + .results + .into_iter() + .map(|r| -> Result { + let verdict = match check_result::CheckVerdict::try_from(r.verdict) + { + Ok(check_result::CheckVerdict::CvFail) => { + check::CheckVerdict::Fail + } + Ok(check_result::CheckVerdict::CvFixed) => { + check::CheckVerdict::Fixed + } + Ok(check_result::CheckVerdict::CvSkip) => { + check::CheckVerdict::Skip + } + Ok(check_result::CheckVerdict::CvPass) => { + check::CheckVerdict::Pass + } + Ok(check_result::CheckVerdict::CvUnspecified) | Err(_) => { + return Err(Error::Other( + anyhow!("invalid check verdict value {}", r.verdict) + .into(), + )); + } + }; + Ok(check::CheckResult { + check: r.check.into(), + verdict, + err: r.errors, + }) + }) + .collect(); + if tx.unbounded_send((obj, check_results)).is_err() { + break; + } + } + None => {} + } + } + }); + + Ok(rx) + } +} + +/// Arguments for a remote check invocation. +pub struct CheckArgs { + pub filter_names: Vec, + pub skip_checkers: Vec, + pub packages: bool, + pub profiles: bool, + pub harnesses: bool, + pub remote_cache_gcs_bucket: Option, } /// The configuration describing an execution in a remote environment. @@ -496,6 +650,14 @@ mod tests { ) -> Result, tonic::Status> { unimplemented!() } + + type CheckStream = tokio_stream::Empty>; + async fn check( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + unimplemented!() + } } #[tokio::test] @@ -654,6 +816,14 @@ mod tests { .collect(); Ok(tonic::Response::new(tokio_stream::iter(responses))) } + + type CheckStream = tokio_stream::Empty>; + async fn check( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + unimplemented!() + } } #[tokio::test] @@ -751,6 +921,14 @@ mod tests { ) -> Result, tonic::Status> { unimplemented!() } + + type CheckStream = tokio_stream::Empty>; + async fn check( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + unimplemented!() + } } #[tokio::test] @@ -783,4 +961,141 @@ mod tests { "expected 'success' in stdout, got: {output}" ); } + + struct CheckMockServer; + + #[tonic::async_trait] + impl RemoteExecutionService for CheckMockServer { + async fn create_env( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + unimplemented!() + } + + type ExecStream = tokio_stream::Empty>; + async fn exec( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + unimplemented!() + } + + type OrchestrateBuildStream = + tokio_stream::Empty>; + async fn orchestrate_build( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + unimplemented!() + } + + type DownloadStream = tokio_stream::Empty>; + async fn download( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + unimplemented!() + } + + type CheckStream = + tokio_stream::Iter>>; + async fn check( + &self, + request: tonic::Request>, + ) -> Result, tonic::Status> { + use check_message::Msg; + let mut stream = request.into_inner(); + + // First message should be the request. + let first = stream.next().await.unwrap().unwrap(); + let Msg::Request(req) = first.msg.unwrap() else { + panic!("expected CheckRequest as first message"); + }; + assert!(!req.client_id.is_empty()); + assert!(req.packages); + + // Collect file chunks and unpack the tarball. + let mut file_data = Vec::new(); + while let Some(msg) = stream.next().await { + let msg = msg.unwrap(); + let Msg::Chunk(data) = msg.msg.unwrap() else { + panic!("expected Chunk message"); + }; + file_data.extend_from_slice(&data); + } + + let dst_dir = tempfile::tempdir().unwrap(); + common::archive::extract_compressed_tar( + std::io::Cursor::new(&file_data), + common::archive::Compression::Zstd, + dst_dir.path(), + None, + ) + .unwrap(); + + // Verify the streamed file arrived intact. + let content = std::fs::read_to_string(dst_dir.path().join("test.txt")).unwrap(); + assert_eq!(content, "check me"); + + // Send back fake check results. + let responses = vec![Ok(CheckResponse { + msg: Some(check_response::Msg::Check(check_response::Check { + obj: Some(CheckObject { + msg: Some(check_object::Msg::Package("test-pkg".into())), + }), + results: vec![ + remote_proto::res::CheckResult { + check: "parse".into(), + verdict: check_result::CheckVerdict::CvPass.into(), + errors: vec![], + }, + remote_proto::res::CheckResult { + check: "fmt".into(), + verdict: check_result::CheckVerdict::CvFail.into(), + errors: vec!["bad formatting".into()], + }, + ], + })), + })]; + + Ok(tonic::Response::new(tokio_stream::iter(responses))) + } + } + + #[tokio::test] + async fn check_streams_and_returns_results() { + // Create a temp directory with a test file. + let src_dir = tempfile::tempdir().unwrap(); + std::fs::write(src_dir.path().join("test.txt"), b"check me").unwrap(); + + let svc = RemoteExecutionServiceServer::new(CheckMockServer); + let graph = Graph::new(); + let mut client = Client::new(RESClient::new(svc), &graph); + + let args = CheckArgs { + filter_names: vec![], + skip_checkers: vec![], + packages: true, + profiles: false, + harnesses: false, + remote_cache_gcs_bucket: None, + }; + + let mut rx = client.check(src_dir.path(), &args).await.unwrap(); + + // First (and only) streamed result. + let (obj, checks) = rx.next().await.expect("expected a check result"); + let checks = checks.unwrap(); + assert_eq!(obj, check::CheckObj::Package("test-pkg".into())); + assert_eq!(checks.len(), 2); + assert!(matches!(checks[0].verdict, check::CheckVerdict::Pass)); + assert_eq!(checks[0].check.as_ref(), "parse"); + assert!(matches!(checks[1].verdict, check::CheckVerdict::Fail)); + assert_eq!(checks[1].check.as_ref(), "fmt"); + assert_eq!(checks[1].err, vec!["bad formatting"]); + + // Stream should be exhausted. + assert!(rx.next().await.is_none()); + } } diff --git a/crates/remote-proto/build.rs b/crates/remote-proto/build.rs index 5266e8219f..22807c0090 100644 --- a/crates/remote-proto/build.rs +++ b/crates/remote-proto/build.rs @@ -8,6 +8,7 @@ fn main() -> Result<()> { "protos/res/create_env.proto", "protos/res/download.proto", "protos/res/task.proto", + "protos/res/check.proto", "protos/res/remote_execution_service.proto", ], &["protos/", "protos/res"], diff --git a/crates/remote-proto/protos/res/check.proto b/crates/remote-proto/protos/res/check.proto new file mode 100644 index 0000000000..a34f880187 --- /dev/null +++ b/crates/remote-proto/protos/res/check.proto @@ -0,0 +1,82 @@ +syntax = "proto3"; +package res; + +import "streams.proto"; + +// The wire format for messages streamed to the server to run checks. +message CheckMessage { + oneof msg { + // The initiation message of the stream. May only be sent as the first message. + CheckRequest request = 1; + // A chunk of data for the files stream. + bytes chunk = 2; + } +} + +// A request to RES to run some checks. +message CheckRequest { + // A client-provided identifier for the build. + string client_id = 1; + // Describes the stream of the repository that will follow this message with idx == 0. + StreamConfig files_stream = 2; + + // Corresponds to CheckCtx::filter_names. + repeated string filter_names = 3; + // Corresponds to CheckCtx::skip_checkers. + repeated string skip_checkers = 4; + // Run checks over packages. + bool packages = 5; + // Run checks over profiles. + bool profiles = 6; + // Run checks over harnesses. + bool harnesses = 7; + + // Configure the provided GCS bucket name as the remote cache. + optional string remote_cache_gcs_bucket = 8; +} + +// Something that was checked. +message CheckObject { + oneof msg { + string package = 1; + string profile = 2; + string harness = 3; + } +} + +// Something that was checked. +message CheckResult { + string check = 1; + enum CheckVerdict { + CV_UNSPECIFIED = 0; + // The check failed. + CV_FAIL = 1; + // The check fixed itself. We include this value for parity with CheckVerdict, but it should + // never happen on the wire. + CV_FIXED = 2; + // The check was skipped. + CV_SKIP = 3; + // The check passed. + CV_PASS = 4; + } + CheckVerdict verdict = 2; + repeated string errors = 3; +} + +// A packet of information streamed back from RES while running some checks. +message CheckResponse { + message Error { + string msg = 1; + } + message Check { + // Identifies what was checked. + CheckObject obj = 1; + // Results from running checks against the object. + repeated CheckResult results = 2; + } + + oneof msg { + Error error = 1; + Check check = 2; + } +} diff --git a/crates/remote-proto/protos/res/remote_execution_service.proto b/crates/remote-proto/protos/res/remote_execution_service.proto index cae41759b9..4c77cc4b2d 100644 --- a/crates/remote-proto/protos/res/remote_execution_service.proto +++ b/crates/remote-proto/protos/res/remote_execution_service.proto @@ -6,6 +6,7 @@ import "res/create_env.proto"; import "res/download.proto"; import "res/orchestrate_build.proto"; import "res/task.proto"; +import "res/check.proto"; service RemoteExecutionService { // Creates an environment with an optional worktree and optional graph. @@ -18,4 +19,7 @@ service RemoteExecutionService { // Downloads a build. rpc Download(DownloadRequest) returns (stream DownloadResponse) {} + + // Runs checkers against a repository. + rpc Check(stream CheckMessage) returns (stream CheckResponse) {} }