choir_queue/conform.rs
1//! The D18 executor conformance suite, runnable against a helper this
2//! repository cannot build.
3//!
4//! The assertions themselves used to live only in
5//! `choir-queue/tests/it/executor.rs`, which gates the four backends
6//! cargo can link. That is every backend except the one the seam was
7//! carved out for: a Firecracker or Cloud Hypervisor driver needs KVM,
8//! so it is written and run on a Linux machine, and a suite it cannot
9//! be pointed at gates nothing. Same checks, same order, in the library
10//! instead, with `choir-ci-conform` as the caller that takes a helper
11//! argv.
12//!
13//! [`conform`] reports rather than panics. A remote helper is usually
14//! wrong in more than one way at once, and a suite that stops at the
15//! first assertion costs a round trip per defect. The one exception is
16//! the handshake: nothing after it means anything if the far side is
17//! not the protocol we speak, so its failure skips the rest rather
18//! than producing seven derived ones.
19//!
20//! # Examples
21//!
22//! ```no_run
23//! use choir_queue::conform::{conform, Fixtures, Outcome};
24//! use choir_queue::executor::Job;
25//! use choir_queue::remote::ProtocolRunner;
26//!
27//! let subject = choir_hash::ContentHash::blake3(b"conform");
28//! let sh = |script: &str| {
29//! Job::new(subject.clone(), vec!["/bin/sh".into(), "-c".into(), script.into()])
30//! };
31//! let mut slow = sh("sleep 30");
32//! slow.deadline = std::time::Duration::from_millis(100);
33//! let checks = conform(
34//! &mut ProtocolRunner::new(vec!["choir-ci-local".to_string()]),
35//! Fixtures {
36//! passing: sh("exit 0"),
37//! failing: sh("exit 3"),
38//! erroring: Job::new(subject.clone(), vec!["/nonexistent/helper".into()]),
39//! slow,
40//! in_directory: None,
41//! },
42//! );
43//! assert!(checks.iter().all(|c| !matches!(c.outcome, Outcome::Failed(_))));
44//! ```
45
46use crate::executor::{CiExecutor, Job, Verdict, PROTOCOL};
47
48/// Jobs meaningful to one backend.
49///
50/// Supplied by the caller because "a command that exits 3" is spelled
51/// differently in a VM than in a subprocess, which is exactly what the
52/// seam abstracts. The checks are about the verdicts, never about the
53/// commands that produced them.
54pub struct Fixtures {
55 /// A command that exits zero.
56 pub passing: Job,
57 /// A command that exits nonzero.
58 pub failing: Job,
59 /// A job the provider cannot start at all.
60 pub erroring: Job,
61 /// A job that outlives its deadline.
62 pub slow: Job,
63 /// A job that passes only if the executor ran it in
64 /// [`Job::directory`], and fails otherwise. `None` for a backend
65 /// with no directory to honor -- an isolation boundary the caller's
66 /// filesystem does not cross is the ordinary case, and skipping is
67 /// the honest answer rather than a failure.
68 pub in_directory: Option<Job>,
69}
70
71/// What one check found.
72#[derive(Debug, Clone, PartialEq, Eq)]
73pub enum Outcome {
74 /// The executor satisfied the check.
75 Passed,
76 /// It did not, and this is what was expected instead.
77 Failed(String),
78 /// The check did not run, and this is why. Never a pass: a report
79 /// naming what it could not establish is the point of the variant.
80 Skipped(String),
81}
82
83/// One named check and its outcome.
84#[derive(Debug, Clone, PartialEq, Eq)]
85pub struct Check {
86 /// Stable name, so a helper's author can be told which one failed.
87 pub name: &'static str,
88 /// What it found.
89 pub outcome: Outcome,
90}
91
92impl std::fmt::Display for Check {
93 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
94 match &self.outcome {
95 Outcome::Passed => write!(f, "ok {}", self.name),
96 Outcome::Failed(why) => write!(f, "FAILED {}: {why}", self.name),
97 Outcome::Skipped(why) => write!(f, "skipped {}: {why}", self.name),
98 }
99 }
100}
101
102/// Every check an implementation of [`CiExecutor`] must satisfy.
103///
104/// Returns one [`Check`] per rule, in a fixed order, whatever happens.
105/// The caller decides what a failure means: the in-tree suite asserts
106/// none, `choir-ci-conform` prints them and exits nonzero.
107#[must_use]
108pub fn conform(ci: &mut dyn CiExecutor, f: Fixtures) -> Vec<Check> {
109 let mut checks = Vec::new();
110
111 // 1. The handshake precedes work and pins the protocol.
112 let handshake = match ci.info() {
113 Err(e) => Err(format!("the handshake did not complete: {e}")),
114 Ok(info) if info.protocol != PROTOCOL => Err(format!(
115 "executor `{}` speaks protocol {} and we speak {PROTOCOL}",
116 info.name, info.protocol
117 )),
118 Ok(info) if info.name.is_empty() => Err("an executor must name itself".to_string()),
119 Ok(_) => Ok(()),
120 };
121 let refused = handshake.is_err();
122 checks.push(check("handshake", handshake));
123 if refused {
124 // Nothing after this is evidence about anything.
125 for name in [
126 "empty-batch",
127 "passed",
128 "failed",
129 "errored",
130 "timed-out",
131 "alignment",
132 "directory",
133 ] {
134 checks.push(Check {
135 name,
136 outcome: Outcome::Skipped("the handshake failed".to_string()),
137 });
138 }
139 return checks;
140 }
141
142 // 2. An empty batch is an empty answer, not an error. The train can
143 // legitimately be empty and that must not read as a fault.
144 checks.push(check(
145 "empty-batch",
146 match ci.run(&[]) {
147 Err(e) => Err(format!("an empty batch must not error: {e}")),
148 Ok(v) if v.is_empty() => Ok(()),
149 Ok(v) => Err(format!("an empty batch must answer nothing, got {v:?}")),
150 },
151 ));
152
153 // 3. A command that succeeds passes.
154 checks.push(check(
155 "passed",
156 one(ci, &f.passing).and_then(|v| match v {
157 Verdict::Passed => Ok(()),
158 other => Err(format!("a zero exit must be Passed, got {other:?}")),
159 }),
160 ));
161
162 // 4. A command that exits nonzero is `Failed` -- about the change --
163 // and is the only verdict permitted to evict.
164 checks.push(check(
165 "failed",
166 one(ci, &f.failing).and_then(|v| match v {
167 Verdict::Failed { .. } if v.evicts() && v.is_conclusive() => Ok(()),
168 Verdict::Failed { .. } => {
169 Err("a test failure must evict and must be conclusive".to_string())
170 }
171 other => Err(format!("a nonzero exit must be Failed, got {other:?}")),
172 }),
173 ));
174
175 // 5. A provider that cannot start the job is `Errored` -- about us --
176 // and must NOT evict. This is the distinction the whole seam
177 // exists for, so it is checked rather than assumed.
178 checks.push(check(
179 "errored",
180 one(ci, &f.erroring).and_then(|v| match v {
181 Verdict::Errored { .. } if !v.evicts() && !v.is_conclusive() => Ok(()),
182 Verdict::Errored { .. } => {
183 Err("a provider fault must never evict and is not conclusive".to_string())
184 }
185 other => Err(format!(
186 "a provider fault must be Errored, not a verdict about the change, got {other:?}"
187 )),
188 }),
189 ));
190
191 // 6. A job past its deadline is `TimedOut`, and also does not evict:
192 // a job can time out because the executor was oversubscribed.
193 checks.push(check(
194 "timed-out",
195 one(ci, &f.slow).and_then(|v| match v {
196 Verdict::TimedOut if !v.evicts() => Ok(()),
197 Verdict::TimedOut => Err("a timeout must never evict a change".to_string()),
198 other => Err(format!(
199 "a job past its deadline must be TimedOut, got {other:?}"
200 )),
201 }),
202 ));
203
204 // 7. Index alignment. A provider that reorders or drops has
205 // attributed one change's result to another, and the batch call
206 // is worthless without this.
207 let batch = vec![f.passing.clone(), f.failing.clone(), f.passing.clone()];
208 checks.push(check(
209 "alignment",
210 match ci.run(&batch) {
211 Err(e) => Err(format!("the mixed batch produced no verdicts: {e}")),
212 Ok(out) if out.len() != batch.len() => Err(format!(
213 "one verdict per job: {} jobs answered by {}",
214 batch.len(),
215 out.len()
216 )),
217 Ok(out)
218 if out[0] == Verdict::Passed
219 && matches!(out[1], Verdict::Failed { .. })
220 && out[2] == Verdict::Passed =>
221 {
222 Ok(())
223 }
224 Ok(out) => Err(format!(
225 "pass/fail/pass came back as {out:?}; a reordered batch attributes \
226 one change's result to another"
227 )),
228 },
229 ));
230
231 // 8. A provider that can run on a caller's checkout runs it in the
232 // directory the job names. The probe passes only from inside
233 // that directory, so an executor that ignores the field returns
234 // `Failed` -- a well-formed verdict about the wrong tree, which
235 // is the failure mode that made this worth a protocol bump.
236 checks.push(match f.in_directory {
237 None => Check {
238 name: "directory",
239 outcome: Outcome::Skipped(
240 "no probe job supplied; this executor was not claimed to honor \
241 Job::directory"
242 .to_string(),
243 ),
244 },
245 Some(job) => check(
246 "directory",
247 one(ci, &job).and_then(|v| match v {
248 Verdict::Passed => Ok(()),
249 other => Err(format!(
250 "the job did not run in the directory it named, got {other:?}"
251 )),
252 }),
253 ),
254 });
255
256 checks
257}
258
259/// One job, one verdict, with the shapes that are not a verdict at all
260/// flattened into the same failure string.
261fn one(ci: &mut dyn CiExecutor, job: &Job) -> Result<Verdict, String> {
262 match ci.run(std::slice::from_ref(job)) {
263 Err(e) => Err(format!("the call produced no verdicts: {e}")),
264 Ok(v) => v
265 .into_iter()
266 .next()
267 .ok_or_else(|| "one job was answered by no verdict".to_string()),
268 }
269}
270
271fn check(name: &'static str, result: Result<(), String>) -> Check {
272 Check {
273 name,
274 outcome: match result {
275 Ok(()) => Outcome::Passed,
276 Err(why) => Outcome::Failed(why),
277 },
278 }
279}