willis: Reduce the number of parallelism.
Seems like running too many processes at one time leaves kernel to
thrash and not very efficient. My benchmark on cloudtop on chrome os
code base on hot caches was:
almost unlimited (original code)
real 0m2.536s
user 0m8.089s
sys 0m34.078s
2* numcpus == 192
real 0m2.428s
user 0m8.692s
sys 0m32.967s
1* numcpus == 96
real 0m2.358s
user 0m8.858s
sys 0m42.823s
BUG=None
TEST=Run willis
Change-Id: Ic8d1d5c288bf5c3e55d6a54817a9e739e9db04ea
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/dev-util/+/2340446
Reviewed-by: Vadim Bendebury <vbendeb@chromium.org>
Commit-Queue: Junichi Uekawa <uekawa@chromium.org>
Tested-by: Junichi Uekawa <uekawa@chromium.org>
diff --git a/contrib/willis.go b/contrib/willis.go
index c6406c7..9e01754 100644
--- a/contrib/willis.go
+++ b/contrib/willis.go
@@ -27,8 +27,8 @@
"path"
"path/filepath"
"regexp"
+ "runtime"
"sort"
- "strconv"
"strings"
"sync"
"time"
@@ -344,24 +344,6 @@
}
-// getMaxGoCount - suggest maximum number of concurrent go routines.
-// Determine current limit of number of open files and return the suggested
-// maximum number of concurrent go routines. Conservatively keep it at 10% of
-// the number of open files limit.
-func getMaxGoCount() (int, error) {
- stdout, _, err := runCommand("sh", "-c", "ulimit -Sn")
- if err != nil {
- return 0, err
- }
- limit, err := strconv.Atoi(stdout)
- if err != nil {
- return 0, err
- }
- // This asssumes that the max number of opened files limit exceeds 10,
- // which is deemed a very reasonable assumption.
- return (limit / 10), nil
-}
-
func main() {
repoRoot, err := findRepoRoot()
if err != nil {
@@ -396,11 +378,10 @@
os.Exit(1)
}
- maxGoCount, err := getMaxGoCount()
- if err != nil {
- fmt.Fprintf(os.Stderr, "Failed to get max go routine count: %v\n", err)
- os.Exit(1)
- }
+ // Use the number of cores as number of goroutines. Because we
+ // are fork/exec multiple instances, exceeding the number of
+ // cores does not give us much gain.
+ maxGoCount := runtime.NumCPU()
repoList := strings.Split(repos, "\n")
