changelog-webapp: Added sysctl value changes
The webapp now can fetch sysctl artifacts in GCS and
find the difference of runtime sysctl values between
two builds. Artifacts bucket is stored in project
secrets. Insignificant sysctl value changes are excluded.
BUG=b/159153192
TEST=go test and deploying on APP engine
Change-Id: I1601d5358381188f760ffa5730065ef841f8c953
Reviewed-on: https://cos-review.googlesource.com/c/cos/tools/+/16010
Cloud-Build: GCB Service account <228075978874@cloudbuild.gserviceaccount.com>
Reviewed-by: Arnav Kansal <rnv@google.com>
Tested-by: He Gao <hegao@google.com>
diff --git a/src/cmd/changelog-webapp/app.yaml b/src/cmd/changelog-webapp/app.yaml
index cb2fb49..74c7692 100644
--- a/src/cmd/changelog-webapp/app.yaml
+++ b/src/cmd/changelog-webapp/app.yaml
@@ -8,10 +8,12 @@
COS_CHANGELOG_CLIENT_SECRET_NAME: "cos-changelog-client-secret"
COS_CHANGELOG_SESSION_SECRET_NAME: "cos-changelog-session-secret"
COS_CHANGELOG_OAUTH_CALLBACK_NAME: "cos-changelog-oauth-callback"
+ COS_CHANGELOG_ARTIFACTS_BUCKET_NAME: "cos-changelog-artifacts-bucket"
# Webpage configuration
STATIC_BASE_PATH: "src/cmd/changelog-webapp/static/"
CHANGELOG_QUERY_SIZE: "50"
+ BOARD_NAME: "lakitu"
# External sources
COS_EXTERNAL_GERRIT_INSTANCE: "https://cos-review.googlesource.com"
diff --git a/src/cmd/changelog-webapp/controllers/pageHandlers.go b/src/cmd/changelog-webapp/controllers/pageHandlers.go
index 8f943c8..d848740 100644
--- a/src/cmd/changelog-webapp/controllers/pageHandlers.go
+++ b/src/cmd/changelog-webapp/controllers/pageHandlers.go
@@ -46,6 +46,8 @@
externalGoBInstance string
externalManifestRepo string
envQuerySize string
+ envBoard string
+ artifactsBucket string
staticBasePath string
indexTemplate *template.Template
@@ -83,10 +85,16 @@
if err != nil {
log.Fatalf("Failed to retrieve secret for CROSLAND_NAME with key name %s\n%v", os.Getenv("CROSLAND_NAME"), err)
}
+ artifactsBucket, err = getSecret(client, os.Getenv("COS_CHANGELOG_ARTIFACTS_BUCKET_NAME"))
+ if err != nil {
+ log.Fatalf("Failed to retrieve secret for COS_CHANGELOG_ARTIFACTS_BUCKET_NAME with key name %s\n%v", os.Getenv("COS_CHANGELOG_ARTIFACTS_BUCKET_NAME"), err)
+ }
+
externalGerritInstance = os.Getenv("COS_EXTERNAL_GERRIT_INSTANCE")
externalFallbackGerritInstance = os.Getenv("COS_EXTERNAL_FALLBACK_GERRIT_INSTANCE")
externalGoBInstance = os.Getenv("COS_EXTERNAL_GOB_INSTANCE")
externalManifestRepo = os.Getenv("COS_EXTERNAL_MANIFEST_REPO")
+ envBoard = os.Getenv("BOARD_NAME")
envQuerySize = getIntVerifiedEnv("CHANGELOG_QUERY_SIZE")
staticBasePath = os.Getenv("STATIC_BASE_PATH")
indexTemplate = template.Must(template.ParseFiles(staticBasePath + "templates/index.html"))
@@ -106,11 +114,22 @@
}
type changelogPage struct {
- Source string
- Target string
- QuerySize string
- RepoTables []*repoTable
- Internal bool
+ Source string
+ Target string
+ SourceBoard string
+ SourceMilestone string
+ TargetBoard string
+ TargetMilestone string
+ QuerySize string
+ RepoTables []*repoTable
+ Internal bool
+ Sysctl sysctlChanges
+}
+
+type sysctlChanges struct {
+ Changes [][]string
+ NotFound string
+ NotEmpty bool
}
type findBuildPage struct {
@@ -319,6 +338,16 @@
}
source := r.FormValue("source")
target := r.FormValue("target")
+ sourceMilestone := r.FormValue("source-milestone")
+ targetMilestone := r.FormValue("target-milestone")
+ sourceBoard := r.FormValue("source-board")
+ if sourceBoard == "" {
+ sourceBoard = envBoard
+ }
+ targetBoard := r.FormValue("target-board")
+ if targetBoard == "" {
+ targetBoard = envBoard
+ }
// If no source/target values specified in request, display empty changelog page
if source == "" || target == "" {
err = changelogTemplate.Execute(w, &changelogPage{QuerySize: envQuerySize, Internal: true})
@@ -356,6 +385,26 @@
Removals: removed,
Internal: internal,
})
+ page.SourceMilestone = sourceMilestone
+ page.SourceBoard = sourceBoard
+ page.TargetMilestone = targetMilestone
+ page.TargetBoard = targetBoard
+
+ var foundSource, foundTarget bool
+ page.Sysctl.Changes, foundSource, foundTarget = changelog.GetSysctlDiff(artifactsBucket, sourceBoard,
+ sourceMilestone, source, targetBoard, targetMilestone, target)
+ page.Sysctl.NotEmpty = false
+ if !foundSource {
+ page.Sysctl.NotFound += fmt.Sprintf("sysctl file for %s-%s-%s not found.<br>", sourceBoard, sourceMilestone, source)
+ page.Sysctl.NotEmpty = true
+ }
+ if !foundTarget {
+ page.Sysctl.NotFound += fmt.Sprintf("sysctl file for %s-%s-%s not found.<br>", targetBoard, targetMilestone, target)
+ page.Sysctl.NotEmpty = true
+ }
+ if len(page.Sysctl.Changes) > 0 {
+ page.Sysctl.NotEmpty = true
+ }
err = changelogTemplate.Execute(w, page)
if err != nil {
log.Errorf("error executing changelog template: %v", err)
diff --git a/src/cmd/changelog-webapp/static/css/changelog.css b/src/cmd/changelog-webapp/static/css/changelog.css
index 3140f47..38db3b7 100644
--- a/src/cmd/changelog-webapp/static/css/changelog.css
+++ b/src/cmd/changelog-webapp/static/css/changelog.css
@@ -128,4 +128,8 @@
.build-info-link {
font-size: 15;
+}
+
+.sysctl{
+ text-align: center;
}
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/static/templates/changelog.html b/src/cmd/changelog-webapp/static/templates/changelog.html
index aeca7c9..ca31be6 100644
--- a/src/cmd/changelog-webapp/static/templates/changelog.html
+++ b/src/cmd/changelog-webapp/static/templates/changelog.html
@@ -18,7 +18,7 @@
</div>
<div class="main">
<h1>Changelog</h1>
- <p class="feature-info">Retrieve a list of commits between two
+ <p class="feature-info">Retrieve a list of commits and sysctl changes between two
Container-Optimized OS builds.<br>
Example Input: <b>cos-rc-85-13310-1034-0</b> or <b>13310.1034.0</b>
</p>
@@ -43,8 +43,37 @@
<input type="text" class="target" name="target" placeholder="Image Name or Build Number" required>
{{end}}
<input type="hidden" name="n" value={{.QuerySize}}>
+ </div>
+ <div class="text">
+ <label>From </label>
+ {{if (ne .SourceMilestone "")}}
+ <input type="text" class="source" name="source-milestone" placeholder="Milestone Number" value={{.SourceMilestone}} required>
+ {{else}}
+ <input type="text" class="source" name="source-milestone" placeholder="Milestone Number" required>
+ {{end}}
+ <label> to </label>
+ {{if (ne .TargetMilestone "")}}
+ <input type="text" class="target" name="target-milestone" placeholder="Milestone Number" value={{.TargetMilestone}} required>
+ {{else}}
+ <input type="text" class="target" name="target-milestone" placeholder="Milestone Number" required>
+ {{end}}
<input class="submit" type="submit" value="Submit"><br>
</div>
+ <div class="text">
+ <label>From </label>
+ {{if (ne .SourceBoard "")}}
+ <input type="text" class="source" name="source-board" placeholder="Board Name" value={{.SourceBoard}}>
+ {{else}}
+ <input type="text" class="source" name="source-board" placeholder="Board Name">
+ {{end}}
+ <label> to </label>
+ {{if (ne .TargetBoard "")}}
+ <input type="text" class="target" name="target-board" placeholder="Board Name" value={{.TargetBoard}}>
+ {{else}}
+ <input type="text" class="target" name="target-board" placeholder="Board Name">
+ {{end}}
+ The default board is "lakitu".
+ </div>
<div class="radio">
{{if .Internal}}
<label>
@@ -146,6 +175,25 @@
</a>
{{end}}
{{end}}
+ {{if .Sysctl.NotEmpty}}
+ <h2>Runtime Sysctl Changes:</h2>
+ <table class="repo-table">
+ <tr>
+ <th class="sysctl">Sysctl</th>
+ <th class="sysctl">Old Value</th>
+ <th class="sysctl">New Value</th>
+ </tr>
+ {{range $sysctl := .Sysctl.Changes}}
+ <tr>
+ <td>{{index $sysctl 0}}</td>
+ <td class="sysctl removal">{{index $sysctl 1}}</td>
+ <td class="sysctl addition">{{index $sysctl 2}}</td>
+ </tr>
+ {{end}}
+ </table>
+ <div>{{.Sysctl.NotFound}}</div>
+ {{end}}
+ </table>
</div>
</body>
</html>
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/static/templates/readme.html b/src/cmd/changelog-webapp/static/templates/readme.html
index 72a3c45..ce8b0ee 100644
--- a/src/cmd/changelog-webapp/static/templates/readme.html
+++ b/src/cmd/changelog-webapp/static/templates/readme.html
@@ -62,6 +62,13 @@
go/crosland
</a>.
</p>
+ <p>
+ It also shows the sysctl changes between two builds if milestone
+ numbers are correctly provided and sysctl values have been dumped for these builds.
+ </p>
+ <p>
+ Example: Milestone Number = 89, Build Number = 16108.403.22
+ </p>
<h2>Find Build</h2>
<p>
This feature allows you to locate the first build containing a desired
diff --git a/src/pkg/changelog/changelog.go b/src/pkg/changelog/changelog.go
index 3d6fea3..110bd52 100644
--- a/src/pkg/changelog/changelog.go
+++ b/src/pkg/changelog/changelog.go
@@ -15,8 +15,9 @@
// This package generates a changelog based on the commit history between
// two build numbers. The changelog consists of two outputs - the commits
// added to the target build that aren't present in the source build, and the
-// commits in the source build that aren't present in the target build. This
-// package uses concurrency to improve performance.
+// commits in the source build that aren't present in the target build. It
+// also finds the sysctl value changes between two builds by fetching artifacts
+// from GCS. This package uses concurrency to improve performance.
//
// This package uses Gitiles to request information from a Git on Borg instance.
// To generate a changelog, the package first retrieves the the manifest files for
@@ -30,11 +31,16 @@
package changelog
import (
+ "context"
"errors"
+ "fmt"
+ "io/ioutil"
"net/http"
"regexp"
+ "sort"
"strings"
+ "cloud.google.com/go/storage"
"cos.googlesource.com/cos/tools.git/src/pkg/utils"
"github.com/beevik/etree"
@@ -298,7 +304,112 @@
}
}
outputChan <- additionsResult{Additions: repoCommits}
- return
+}
+
+// getSysctlDiff finds sysctl difference between the two builds.
+// Returns a list of change lists:[[name, old-value, new-value], ...]
+func GetSysctlDiff(bucket, sourceBoard, sourceMilestone, source, targetBoard, targetMilestone, target string) (
+ [][]string, bool, bool) {
+ sourceBuildNum, targetBuildNum := resolveImageName(source), resolveImageName(target)
+ sourceChan := make(chan map[string]string)
+ targetChan := make(chan map[string]string)
+ ctx := context.Background()
+ client, err := storage.NewClient(ctx)
+ if err != nil {
+ log.Errorf("failed to create storage client (error: %s)", err)
+ return [][]string{}, false, false
+ }
+ go fetchSysctlToMap(fmt.Sprintf("%s/%s-release/R%s-%s",
+ bucket, sourceBoard, sourceMilestone, sourceBuildNum), sourceChan, client, ctx)
+ go fetchSysctlToMap(fmt.Sprintf("%s/%s-release/R%s-%s",
+ bucket, targetBoard, targetMilestone, targetBuildNum), targetChan, client, ctx)
+ sourceSysctl := <-sourceChan
+ targetSysctl := <-targetChan
+ foundSource := false
+ foundTarget := false
+ // if either one of the sysctl file doesn't exist,
+ // return an empty list.
+ if len(sourceSysctl) > 0 {
+ foundSource = true
+ }
+ if len(targetSysctl) > 0 {
+ foundTarget = true
+ }
+ if !foundSource || !foundTarget {
+ return [][]string{}, foundSource, foundTarget
+ }
+
+ changes := [][]string{}
+ for newName, newValue := range targetSysctl {
+ if oldValue, found := sourceSysctl[newName]; !found {
+ changes = append(changes, []string{newName, "---", newValue})
+ } else if oldValue != newValue {
+ changes = append(changes, []string{newName, oldValue, newValue})
+ }
+ }
+ for oldName, oldValue := range sourceSysctl {
+ if _, found := targetSysctl[oldName]; !found {
+ changes = append(changes, []string{oldName, oldValue, "---"})
+ }
+ }
+
+ sort.SliceStable(changes, func(i, j int) bool {
+ return changes[i][0] < changes[j][0]
+ })
+
+ return changes, foundSource, foundTarget
+}
+
+// fetchSysctlToMap fetches sysctl file from artifacts in GCS created
+// by build-executor and map each line to a <parameter_name: value>
+// pair.
+func fetchSysctlToMap(path string, outputChan chan map[string]string, client *storage.Client, ctx context.Context) {
+ // Some sysctl value changes are insignificant and should not be displayed.
+ sysctlFilter := map[string]bool{
+ "kernel.hostname": true,
+ "kernel.version": true,
+ "fs.dentry-state": true,
+ "fs.file-nr": true,
+ "fs.inode-nr": true,
+ "fs.inode-state": true,
+ "fs.quota.syncs": true,
+ "kernel.ns_last_pid": true,
+ "kernel.pty.nr": true,
+ "kernel.random.boot_id": true,
+ "kernel.random.entropy_avail": true,
+ "kernel.random.uuid": true,
+ "net.netfilter.nf_conntrack_count": true,
+ "kernel.osrelease": true,
+ }
+ outMap := make(map[string]string)
+ defer func() { outputChan <- outMap }()
+ rc, err := client.Bucket(path).Object("sysctl_a.txt").NewReader(ctx)
+ if err != nil {
+ log.Errorf("failed to open %s at %s (error:%s)", "sysctl_a.txt", path, err)
+ return
+ }
+
+ byteBuf, err := ioutil.ReadAll(rc)
+ rc.Close()
+ if err != nil {
+ log.Errorf("failed to read sysctl file (error:%s)", err)
+ return
+ }
+ separator := " = "
+ for _, line := range strings.Split(string(byteBuf), "\n") {
+ parts := strings.Split(line, separator)
+ // Insignificant sysctl parameters are excluded.
+ if _, found := sysctlFilter[parts[0]]; found {
+ continue
+ }
+ // no value for this parameter
+ if len(parts) == 2 && parts[1] == "" {
+ outMap[parts[0]] = "---"
+ } else {
+ // assume the parameter name is before the first separator.
+ outMap[parts[0]] = strings.Join(parts[1:], separator)
+ }
+ }
}
// Changelog generates a changelog between 2 build numbers
