(Arne Babenhauserheide)
2017-04-02: set max iterations set max iterations
diff --git a/examples/benchmark.w b/examples/benchmark.w --- a/examples/benchmark.w +++ b/examples/benchmark.w @@ -64,6 +64,7 @@ define* : benchmark-run-single fun #:key profiler (* 4 loop-num) ;; for fast functions I need to go up rapidly, for slow ones I need to avoid overshooting ;; Define targets for the data aquisition +define max-iterations 1.e5 ;; at most 100k samples, which is more than I typically see define max-relative-uncertainty 0.3 ;; 3 sigma from 0 define min-aggregated-runtime-seconds 1.e-5 ;; 10μs ~ 30k cycles define max-absolute-uncertainty-seconds 1.e-3 ;; 1ms, required to ensure that the model uses the higher values (else they would have huge uncertainties). If you find you need more, use a smaller test case. @@ -74,9 +75,10 @@ define* : benchmark-run fun let* : res : list-ec (: i sampling-steps) : benchmark-run-single fun #:min-seconds min-seconds std : stddev-unbiased-normal res - mean : / (apply + res) (length res) + len : length res + mean : / (apply + res) len ;; pretty-print : list mean '± std min-seconds sampling-steps - if : and {std < {mean * max-relative-uncertainty}} {std < max-absolute-uncertainty-seconds} + if : or {len > max-iterations} : and {std < {mean * max-relative-uncertainty}} {std < max-absolute-uncertainty-seconds} . mean lp (* 2 min-seconds) (* 2 sampling-steps) ;; should decrease σ by factor 2 or √2 (for slow functions) @@ -298,7 +300,6 @@ define* : plot-benchmark-result bench H y⁰-pos : map car bench y⁰ : append-map cdr bench y⁰-stds : list-ec (: i y⁰) : min max-absolute-uncertainty-seconds {max-relative-uncertainty * i} ; enforcing 20% max std in benchmark-run - y⁰-std : list-ref (sort y⁰ <) : round : / (length y⁰) 8 ; lower octile median R : make-covariance-matrix-with-offdiagonals-using-stds y⁰-stds optimized : EnSRF H x^b P y⁰ R y⁰-pos ensemble-member-count x-opt : list-ref optimized 0 @@ -333,7 +334,6 @@ define* : plot-benchmark-result bench H format port "import pylab as pl\nimport matplotlib as mpl\n" format port "y0 = [float(i) for i in '~A'[1:-1].split(' ')]\n" y⁰ format port "ystds = [float(i) for i in '~A'[1:-1].split(' ')]\n" y⁰-stds - format port "yerr = ~A\n" y⁰-std format port "ypos1 = [float(i) for i in '~A'[1:-1].split(' ')]\n" : list-ec (: i y⁰-pos) : first i format port "ypos2 = [float(i) for i in '~A'[1:-1].split(' ')]\n" : list-ec (: i y⁰-pos) : second i format port "yinit = [float(i) for i in '~A'[1:-1].split(' ')]\n" : list-ec (: i y⁰-pos) : H x^b i