Fix benchmark script (#905)

pelletier · Oct 2, 2023 · a76e18e · a76e18e
1 parent dff0c12
commit a76e18e
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 25 deletions.
diff --git a/README.md b/README.md
@@ -179,12 +179,12 @@ Execution time speedup compared to other Go TOML libraries:
         <tr><th>Benchmark</th><th>go-toml v1</th><th>BurntSushi/toml</th></tr>
     </thead>
     <tbody>
-        <tr><td>Marshal/HugoFrontMatter-2</td><td>1.9x</td><td>1.9x</td></tr>
-        <tr><td>Marshal/ReferenceFile/map-2</td><td>1.7x</td><td>1.8x</td></tr>
-        <tr><td>Marshal/ReferenceFile/struct-2</td><td>2.2x</td><td>2.5x</td></tr>
-        <tr><td>Unmarshal/HugoFrontMatter-2</td><td>2.9x</td><td>2.9x</td></tr>
-        <tr><td>Unmarshal/ReferenceFile/map-2</td><td>2.6x</td><td>2.9x</td></tr>
-        <tr><td>Unmarshal/ReferenceFile/struct-2</td><td>4.4x</td><td>5.3x</td></tr>
+        <tr><td>Marshal/HugoFrontMatter-2</td><td>1.9x</td><td>2.2x</td></tr>
+        <tr><td>Marshal/ReferenceFile/map-2</td><td>1.7x</td><td>2.1x</td></tr>
+        <tr><td>Marshal/ReferenceFile/struct-2</td><td>2.2x</td><td>3.0x</td></tr>
+        <tr><td>Unmarshal/HugoFrontMatter-2</td><td>2.9x</td><td>2.7x</td></tr>
+        <tr><td>Unmarshal/ReferenceFile/map-2</td><td>2.6x</td><td>2.7x</td></tr>
+        <tr><td>Unmarshal/ReferenceFile/struct-2</td><td>4.6x</td><td>5.1x</td></tr>
      </tbody>
 </table>
 <details><summary>See more</summary>
@@ -197,17 +197,17 @@ provided for completeness.</p>
         <tr><th>Benchmark</th><th>go-toml v1</th><th>BurntSushi/toml</th></tr>
     </thead>
     <tbody>
-        <tr><td>Marshal/SimpleDocument/map-2</td><td>1.8x</td><td>2.9x</td></tr>
-        <tr><td>Marshal/SimpleDocument/struct-2</td><td>2.7x</td><td>4.2x</td></tr>
-        <tr><td>Unmarshal/SimpleDocument/map-2</td><td>4.5x</td><td>3.1x</td></tr>
-        <tr><td>Unmarshal/SimpleDocument/struct-2</td><td>6.2x</td><td>3.9x</td></tr>
-        <tr><td>UnmarshalDataset/example-2</td><td>3.1x</td><td>3.5x</td></tr>
-        <tr><td>UnmarshalDataset/code-2</td><td>2.3x</td><td>3.1x</td></tr>
-        <tr><td>UnmarshalDataset/twitter-2</td><td>2.5x</td><td>2.6x</td></tr>
-        <tr><td>UnmarshalDataset/citm_catalog-2</td><td>2.1x</td><td>2.2x</td></tr>
-        <tr><td>UnmarshalDataset/canada-2</td><td>1.6x</td><td>1.3x</td></tr>
-        <tr><td>UnmarshalDataset/config-2</td><td>4.3x</td><td>3.2x</td></tr>
-        <tr><td>[Geo mean]</td><td>2.7x</td><td>2.8x</td></tr>
+        <tr><td>Marshal/SimpleDocument/map-2</td><td>1.8x</td><td>2.7x</td></tr>
+        <tr><td>Marshal/SimpleDocument/struct-2</td><td>2.7x</td><td>3.8x</td></tr>
+        <tr><td>Unmarshal/SimpleDocument/map-2</td><td>3.8x</td><td>3.0x</td></tr>
+        <tr><td>Unmarshal/SimpleDocument/struct-2</td><td>5.6x</td><td>4.1x</td></tr>
+        <tr><td>UnmarshalDataset/example-2</td><td>3.0x</td><td>3.2x</td></tr>
+        <tr><td>UnmarshalDataset/code-2</td><td>2.3x</td><td>2.9x</td></tr>
+        <tr><td>UnmarshalDataset/twitter-2</td><td>2.6x</td><td>2.7x</td></tr>
+        <tr><td>UnmarshalDataset/citm_catalog-2</td><td>2.2x</td><td>2.3x</td></tr>
+        <tr><td>UnmarshalDataset/canada-2</td><td>1.8x</td><td>1.5x</td></tr>
+        <tr><td>UnmarshalDataset/config-2</td><td>4.1x</td><td>2.9x</td></tr>
+        <tr><td>geomean</td><td>2.7x</td><td>2.8x</td></tr>
      </tbody>
 </table>
 <p>This table can be generated with <code>./ci.sh benchmark -a -html</code>.</p>

diff --git a/ci.sh b/ci.sh
@@ -152,7 +152,7 @@ bench() {
     fi
 
     export GOMAXPROCS=2
-    nice -n -19 taskset --cpu-list 0,1 go test '-bench=^Benchmark(Un)?[mM]arshal' -count=5 -run=Nothing ./... | tee "${out}"
+    go test '-bench=^Benchmark(Un)?[mM]arshal' -count=10 -run=Nothing ./... | tee "${out}"
     popd
 
     if [ "${branch}" != "HEAD" ]; then
@@ -161,10 +161,12 @@ bench() {
 }
 
 fmktemp() {
-    if mktemp --version|grep GNU >/dev/null; then
-        mktemp --suffix=-$1;
+    if mktemp --version &> /dev/null; then
+	# GNU
+        mktemp --suffix=-$1
     else
-        mktemp -t $1;
+	# BSD
+	mktemp -t $1
     fi
 }
 
@@ -184,12 +186,14 @@ with open(sys.argv[1]) as f:
             lines.append(line.split(','))
 
 results = []
-for line in reversed(lines[1:]):
+for line in reversed(lines[2:]):
+    if len(line) < 8 or line[0] == "":
+        continue
     v2 = float(line[1])
     results.append([
         line[0].replace("-32", ""),
         "%.1fx" % (float(line[3])/v2),  # v1
-        "%.1fx" % (float(line[5])/v2),  # bs
+        "%.1fx" % (float(line[7])/v2),  # bs
     ])
 # move geomean to the end
 results.append(results[0])
@@ -260,10 +264,10 @@ benchmark() {
 
         if [ "$1" = "-html" ]; then
             tmpcsv=`fmktemp csv`
-            benchstat -csv -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt > $tmpcsv
+            benchstat -format csv go-toml-v2.txt go-toml-v1.txt bs-toml.txt > $tmpcsv
             benchstathtml $tmpcsv
         else
-            benchstat -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt
+            benchstat go-toml-v2.txt go-toml-v1.txt bs-toml.txt
         fi
 
         rm -f go-toml-v2.txt go-toml-v1.txt bs-toml.txt