crosperf: replace statistics stdev with pstdev
Behavior and results of statistics.stdev(data) are slightly different from
numpy.std(data) with default "ddof".
The main difference is the divisor which is "N - 1" in stdev vs. "N" in
numpy.std. As a consequence stdev fails with "N=1".
The change replaces stdev with pstdev (population standard deviation)
which is equivalent to numpy.std with default arguments that we were
using.
Added unittest with StdResult testing.
BUG=None
TEST=unittest and crosperf with one iteration passes.
Change-Id: I70c7105e6cabc27437504de16ea27afdd719e552
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2376880
Reviewed-by: George Burgess <gbiv@chromium.org>
Tested-by: Denis Nikitin <denik@chromium.org>
Commit-Queue: Denis Nikitin <denik@chromium.org>
diff --git a/cros_utils/tabulator.py b/cros_utils/tabulator.py
index 27b1c64..1a3fd4a 100644
--- a/cros_utils/tabulator.py
+++ b/cros_utils/tabulator.py
@@ -610,7 +610,7 @@
def _ComputeFloat(self, cell, values, baseline_values):
if self.ignore_min_max:
values = _RemoveMinMax(cell, values)
- cell.value = statistics.stdev(values)
+ cell.value = statistics.pstdev(values)
class CoeffVarResult(NumericalResult):
@@ -624,7 +624,7 @@
if self.ignore_min_max:
values = _RemoveMinMax(cell, values)
if statistics.mean(values) != 0.0:
- noise = abs(statistics.stdev(values) / statistics.mean(values))
+ noise = abs(statistics.pstdev(values) / statistics.mean(values))
else:
noise = 0.0
cell.value = noise
@@ -1498,40 +1498,40 @@
if __name__ == '__main__':
# Run a few small tests here.
- runs = [
- [{
- 'k1': '10',
- 'k2': '12',
- 'k5': '40',
- 'k6': '40',
- 'ms_1': '20',
- 'k7': 'FAIL',
- 'k8': 'PASS',
- 'k9': 'PASS',
- 'k10': '0'
- }, {
- 'k1': '13',
- 'k2': '14',
- 'k3': '15',
- 'ms_1': '10',
- 'k8': 'PASS',
- 'k9': 'FAIL',
- 'k10': '0'
- }],
- [{
- 'k1': '50',
- 'k2': '51',
- 'k3': '52',
- 'k4': '53',
- 'k5': '35',
- 'k6': '45',
- 'ms_1': '200',
- 'ms_2': '20',
- 'k7': 'FAIL',
- 'k8': 'PASS',
- 'k9': 'PASS'
- }],
- ]
+ run1 = {
+ 'k1': '10',
+ 'k2': '12',
+ 'k5': '40',
+ 'k6': '40',
+ 'ms_1': '20',
+ 'k7': 'FAIL',
+ 'k8': 'PASS',
+ 'k9': 'PASS',
+ 'k10': '0'
+ }
+ run2 = {
+ 'k1': '13',
+ 'k2': '14',
+ 'k3': '15',
+ 'ms_1': '10',
+ 'k8': 'PASS',
+ 'k9': 'FAIL',
+ 'k10': '0'
+ }
+ run3 = {
+ 'k1': '50',
+ 'k2': '51',
+ 'k3': '52',
+ 'k4': '53',
+ 'k5': '35',
+ 'k6': '45',
+ 'ms_1': '200',
+ 'ms_2': '20',
+ 'k7': 'FAIL',
+ 'k8': 'PASS',
+ 'k9': 'PASS'
+ }
+ runs = [[run1, run2], [run3]]
labels = ['vanilla', 'modified']
t = GetComplexTable(runs, labels, TablePrinter.CONSOLE)
print(t)
diff --git a/cros_utils/tabulator_test.py b/cros_utils/tabulator_test.py
index 227e2d7..9dd4828 100755
--- a/cros_utils/tabulator_test.py
+++ b/cros_utils/tabulator_test.py
@@ -33,6 +33,20 @@
result.Compute(cell, table[2], table[1])
self.assertTrue(cell.value == float(table[2][0]))
+ def testStdResult(self):
+ table = ['k1', [], ['1', '2']]
+ result = tabulator.StdResult()
+ cell = tabulator.Cell()
+ result.Compute(cell, table[2], table[1])
+ self.assertTrue(cell.value == 0.5)
+
+ def testStdResultOfSampleSize1(self):
+ table = ['k1', [], ['1']]
+ result = tabulator.StdResult()
+ cell = tabulator.Cell()
+ result.Compute(cell, table[2], table[1])
+ self.assertTrue(cell.value == 0.0)
+
def testStringMean(self):
smr = tabulator.StringMeanResult()
cell = tabulator.Cell()
