choir_queue/
envelope.rs

1//! What precision a detector can reach at *our* prevalence (D23).
2//!
3//! Published detector numbers are measured on corpora that are roughly
4//! one-third positive, because they sample same-declaration parallel
5//! edits. Deployed against a mainline where bad merges are well under
6//! one percent, the same detector reports the same recall and a
7//! completely different precision — the false positives are drawn from a
8//! vastly larger pool of negatives.
9//!
10//! That transformation is the difference between "60% recall at 43%
11//! precision, promising" and "one true alarm per 160", so it should be
12//! computed rather than eyeballed. Recall and specificity are properties
13//! of the detector and carry across; precision is a property of the
14//! detector *and the population* and does not.
15//!
16//! # What the prevalence here actually is
17//!
18//! The measured 0.004 is `choir-queue::corpus`'s revert-labelled bad-merge
19//! rate on rust-lang/rust — merges that history took back. That is not
20//! the same event as "merges containing a semantic conflict": it misses
21//! defects fixed forward and includes reverts that were not defects. It
22//! is used because it is the only prevalence anyone has measured on a
23//! real mainline, and because the operational question is about flagging
24//! merges, not about conflicts in the abstract.
25
26/// A detector's prevalence-independent behaviour.
27#[derive(Debug, Clone, Copy, PartialEq)]
28pub struct Detector {
29    /// Fraction of genuinely bad merges it flags. Carries across
30    /// populations.
31    pub recall: f64,
32    /// Fraction of genuinely good merges it flags anyway. Carries across
33    /// populations, and is the number that decides everything at low
34    /// prevalence.
35    pub false_positive_rate: f64,
36}
37
38impl Detector {
39    /// Recovers the prevalence-independent behaviour from a paper's
40    /// reported `precision` and `recall` at that paper's corpus
41    /// `prevalence`.
42    ///
43    /// # Errors
44    ///
45    /// Inputs outside `0.0..=1.0`, a zero precision or recall (nothing to
46    /// recover), or a corpus prevalence of 1.0 (no negatives to have
47    /// produced the reported false positives).
48    pub fn from_reported(precision: f64, recall: f64, prevalence: f64) -> Result<Self, String> {
49        for (name, v) in [
50            ("precision", precision),
51            ("recall", recall),
52            ("prevalence", prevalence),
53        ] {
54            if !(0.0..=1.0).contains(&v) || v.is_nan() {
55                return Err(format!("{name} must be in 0.0..=1.0, got {v}"));
56            }
57        }
58        if precision <= 0.0 || recall <= 0.0 {
59            return Err("precision and recall must be positive to recover a rate".to_string());
60        }
61        if prevalence >= 1.0 {
62            return Err(
63                "a corpus with no negatives cannot yield a false-positive rate".to_string(),
64            );
65        }
66        // true positives per merge, then false positives per merge from
67        // precision = TP / (TP + FP).
68        let tp = prevalence * recall;
69        let fp = tp * (1.0 - precision) / precision;
70        Ok(Self {
71            recall,
72            false_positive_rate: fp / (1.0 - prevalence),
73        })
74    }
75
76    /// Precision this detector reaches at `prevalence`.
77    ///
78    /// Returns 0.0 when it would flag nothing at all, which is the honest
79    /// reading: a detector that never fires has no precision to speak of.
80    #[must_use]
81    pub fn precision_at(&self, prevalence: f64) -> f64 {
82        let tp = prevalence * self.recall;
83        let fp = (1.0 - prevalence) * self.false_positive_rate;
84        if tp + fp <= 0.0 {
85            return 0.0;
86        }
87        tp / (tp + fp)
88    }
89
90    /// False alarms per true alarm at `prevalence` — the number an
91    /// on-call human actually experiences. `None` when it never fires on
92    /// a true positive, so the ratio is undefined rather than infinite.
93    #[must_use]
94    pub fn false_alarms_per_hit(&self, prevalence: f64) -> Option<f64> {
95        let tp = prevalence * self.recall;
96        if tp <= 0.0 {
97            return None;
98        }
99        Some((1.0 - prevalence) * self.false_positive_rate / tp)
100    }
101}
102
103/// The false-positive rate a detector must not exceed to reach
104/// `target_precision` at `prevalence` with `recall`.
105///
106/// This is the build-or-don't-build number: compare it against what the
107/// literature actually achieves, or against a test suite's flake rate.
108///
109/// # Errors
110///
111/// Inputs outside `0.0..=1.0`, or a target precision of 1.0 (which
112/// demands a zero false-positive rate and is not a useful target).
113pub fn required_false_positive_rate(
114    prevalence: f64,
115    recall: f64,
116    target_precision: f64,
117) -> Result<f64, String> {
118    for (name, v) in [
119        ("prevalence", prevalence),
120        ("recall", recall),
121        ("target_precision", target_precision),
122    ] {
123        if !(0.0..=1.0).contains(&v) || v.is_nan() {
124            return Err(format!("{name} must be in 0.0..=1.0, got {v}"));
125        }
126    }
127    if target_precision >= 1.0 {
128        return Err("a target precision of 1.0 demands zero false positives".to_string());
129    }
130    if prevalence >= 1.0 {
131        return Err("prevalence must leave some negatives".to_string());
132    }
133    // precision = TP / (TP + FP) = target  =>  FP = TP * (1 - target) / target
134    let tp = prevalence * recall;
135    let fp = tp * (1.0 - target_precision) / target_precision;
136    Ok(fp / (1.0 - prevalence))
137}
choir_queue/envelope.rs

choir_queue/
envelope.rs
