Make benchmark module less incorrect

1 files changed, 30 insertions, 15 deletions

diff --git a/2018/src/benchmark.rs b/2018/src/benchmark.rs
index 76de9e4..697fa8e 100644
--- a/2018/src/benchmark.rs
+++ b/2018/src/benchmark.rs
@@ -7,13 +7,13 @@ const MIN_TIMES: usize = 4;
 pub struct Bench {
     pub executions: usize,
     pub duration: f64,
-    pub duration_stddev: f64,
+    pub duration_stderr: f64,
 }
 
 impl fmt::Display for Bench {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "{} secs ± {} ({}%)",
-               self.duration, self.duration_stddev, self.duration_stddev / self.duration * 100.0)
+               self.duration, self.duration_stderr, self.duration_stderr / self.duration * 100.0)
     }
 }
 
@@ -51,28 +51,43 @@ pub fn benchmark<A, F: Fn() -> A>(func: F) -> Bench {
     let nparts = if ntimes_guess < 10 {
         ntimes_guess
     } else {
-        std::cmp::max(10, ntimes_guess / 10)
+        std::cmp::max(10, ntimes_guess / 100)
     };
     let nperpart = (ntimes_guess + nparts - 1) / nparts;
 
-    let ntimes = nparts * nperpart;
-
     // eprintln!("BENCH: estimate={} ntimes_guess={} nparts={} nperpart={} ntimes={}", estimate, ntimes_guess, nparts, nperpart, ntimes);
 
-    let mut times = vec![0.0; nparts];
+    let mut times = Vec::new();
 
-    for i in 0..nparts {
-        let start = Instant::now();
-        for _ in 0..nperpart {
-            func();
+    let mut mean = 0.0;
+    let mut stderr = 0.0;
+
+    eprint!("BENCH: Running {} blocks of {} calls...", nparts, nperpart);
+
+    for _ in 0..6 {
+        for _ in 0..nparts {
+            let start = Instant::now();
+            for _ in 0..nperpart {
+                func();
+            }
+            let end = Instant::now();
+
+            times.push(duration_secs(end - start));
+        }
+
+        let partmean = times.iter().map(|&t| t).sum::<f64>() / times.len() as f64;
+        let stddev = (times.iter().map(|&t| (t - partmean) * (t - partmean)).sum::<f64>() / (times.len() - 1) as f64).sqrt();
+        stderr = stddev / (times.len() as f64).sqrt();
+        mean = partmean / nperpart as f64;
+
+        if stderr / mean < 0.05 {
+            break;
         }
-        let end = Instant::now();
 
-        times[i] = duration_secs(end - start);
+        eprint!(" again({},{})", mean, stderr);
     }
 
-    let mean = times.iter().map(|&t| t).sum::<f64>() / ntimes as f64;
-    let stddev = times.iter().map(|t| (t - mean) * (t - mean)).sum::<f64>() / (ntimes - 1) as f64;
+    eprintln!("");
 
-    Bench { executions: 0, duration: mean, duration_stddev: stddev }
+    Bench { executions: times.len() * nperpart, duration: mean, duration_stderr: stderr }
 }