changelog-webapp: Added sysctl value changes
The webapp now can fetch sysctl artifacts in GCS and
find the difference of runtime sysctl values between
two builds. Artifacts bucket and path suffix are stored
in project secrets. Insignificant sysctl value changes
are excluded.
BUG=b/159153192
TEST=go test and deploying on APP engine
Change-Id: Idbebdc5bc04fd4990a59ba8eb1505103734d2703
diff --git a/src/cmd/changelog-webapp/app.yaml b/src/cmd/changelog-webapp/app.yaml
index cb2fb49..7c40e26 100644
--- a/src/cmd/changelog-webapp/app.yaml
+++ b/src/cmd/changelog-webapp/app.yaml
@@ -8,10 +8,13 @@
COS_CHANGELOG_CLIENT_SECRET_NAME: "cos-changelog-client-secret"
COS_CHANGELOG_SESSION_SECRET_NAME: "cos-changelog-session-secret"
COS_CHANGELOG_OAUTH_CALLBACK_NAME: "cos-changelog-oauth-callback"
+ COS_CHANGELOG_ARTIFACTS_BUCKET_NAME: "cos-changelog-artifacts-bucket"
+ COS_CHANGELOG_ARTIFACTS_PATH_SUFFIX_NAME: "cos-changelog-artifacts-path-suffix"
# Webpage configuration
STATIC_BASE_PATH: "src/cmd/changelog-webapp/static/"
CHANGELOG_QUERY_SIZE: "50"
+ BOARD_NAME: "lakitu"
# External sources
COS_EXTERNAL_GERRIT_INSTANCE: "https://cos-review.googlesource.com"
diff --git a/src/cmd/changelog-webapp/controllers/pageHandlers.go b/src/cmd/changelog-webapp/controllers/pageHandlers.go
index 8f943c8..90e1975 100644
--- a/src/cmd/changelog-webapp/controllers/pageHandlers.go
+++ b/src/cmd/changelog-webapp/controllers/pageHandlers.go
@@ -46,6 +46,9 @@
externalGoBInstance string
externalManifestRepo string
envQuerySize string
+ envBoard string
+ artifactsBucket string
+ artifactsPathSuffix string
staticBasePath string
indexTemplate *template.Template
@@ -83,10 +86,20 @@
if err != nil {
log.Fatalf("Failed to retrieve secret for CROSLAND_NAME with key name %s\n%v", os.Getenv("CROSLAND_NAME"), err)
}
+ artifactsBucket, err = getSecret(client, os.Getenv("COS_CHANGELOG_ARTIFACTS_BUCKET_NAME"))
+ if err != nil {
+ log.Fatalf("Failed to retrieve secret for COS_CHANGELOG_ARTIFACTS_BUCKET_NAME with key name %s\n%v", os.Getenv("COS_CHANGELOG_ARTIFACTS_BUCKET_NAME"), err)
+ }
+ artifactsPathSuffix, err = getSecret(client, os.Getenv("COS_CHANGELOG_ARTIFACTS_PATH_SUFFIX_NAME"))
+ if err != nil {
+ log.Fatalf("Failed to retrieve secret for COS_CHANGELOG_ARTIFACTS_PATH_SUFFIX_NAME with key name %s\n%v", os.Getenv("COS_CHANGELOG_ARTIFACTS_PATH_SUFFIX_NAME"), err)
+ }
+
externalGerritInstance = os.Getenv("COS_EXTERNAL_GERRIT_INSTANCE")
externalFallbackGerritInstance = os.Getenv("COS_EXTERNAL_FALLBACK_GERRIT_INSTANCE")
externalGoBInstance = os.Getenv("COS_EXTERNAL_GOB_INSTANCE")
externalManifestRepo = os.Getenv("COS_EXTERNAL_MANIFEST_REPO")
+ envBoard = os.Getenv("BOARD_NAME")
envQuerySize = getIntVerifiedEnv("CHANGELOG_QUERY_SIZE")
staticBasePath = os.Getenv("STATIC_BASE_PATH")
indexTemplate = template.Must(template.ParseFiles(staticBasePath + "templates/index.html"))
@@ -106,11 +119,24 @@
}
type changelogPage struct {
- Source string
- Target string
- QuerySize string
- RepoTables []*repoTable
- Internal bool
+ Source string
+ Target string
+ SourceBoard string
+ SourceMilestone string
+ TargetBoard string
+ TargetMilestone string
+ QuerySize string
+ RepoTables []*repoTable
+ Internal bool
+ Sysctl *sysctlChanges
+}
+
+type sysctlChanges struct {
+ Added []string
+ Changed []string
+ Deleted []string
+ NotFound string
+ NotEmpty bool
}
type findBuildPage struct {
@@ -310,7 +336,7 @@
}
var err error
if err := r.ParseForm(); err != nil {
- err = changelogTemplate.Execute(w, &changelogPage{QuerySize: envQuerySize})
+ err = changelogTemplate.Execute(w, &changelogPage{QuerySize: envQuerySize, Sysctl: new(sysctlChanges)})
if err != nil {
log.Errorf("error executing findbuild template: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -319,9 +345,19 @@
}
source := r.FormValue("source")
target := r.FormValue("target")
+ sourceMilestone := r.FormValue("source milestone")
+ targetMilestone := r.FormValue("target milestone")
+ sourceBoard := r.FormValue("source board")
+ if sourceBoard == "" {
+ sourceBoard = envBoard
+ }
+ targetBoard := r.FormValue("target board")
+ if targetBoard == "" {
+ targetBoard = envBoard
+ }
// If no source/target values specified in request, display empty changelog page
if source == "" || target == "" {
- err = changelogTemplate.Execute(w, &changelogPage{QuerySize: envQuerySize, Internal: true})
+ err = changelogTemplate.Execute(w, &changelogPage{QuerySize: envQuerySize, Internal: true, Sysctl: new(sysctlChanges)})
if err != nil {
log.Errorf("error executing findbuild template: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -356,6 +392,28 @@
Removals: removed,
Internal: internal,
})
+ page.SourceMilestone = sourceMilestone
+ page.SourceBoard = sourceBoard
+ page.TargetMilestone = targetMilestone
+ page.TargetBoard = targetBoard
+
+ page.Sysctl = new(sysctlChanges)
+ var foundSource, foundTarget bool
+ page.Sysctl.Added, page.Sysctl.Changed, page.Sysctl.Deleted,
+ foundSource, foundTarget = changelog.GetSysctlDiff(artifactsBucket, artifactsPathSuffix, sourceBoard,
+ sourceMilestone, source, targetBoard, targetMilestone, target)
+ page.Sysctl.NotEmpty = false
+ if !foundSource {
+ page.Sysctl.NotFound += fmt.Sprintf("sysctl file for %s-%s-%s not found.<br>", sourceBoard, sourceMilestone, source)
+ page.Sysctl.NotEmpty = true
+ }
+ if !foundTarget {
+ page.Sysctl.NotFound += fmt.Sprintf("sysctl file for %s-%s-%s not found.<br>", targetBoard, targetMilestone, target)
+ page.Sysctl.NotEmpty = true
+ }
+ if len(page.Sysctl.Added) > 0 || len(page.Sysctl.Changed) > 0 || len(page.Sysctl.Deleted) > 0 {
+ page.Sysctl.NotEmpty = true
+ }
err = changelogTemplate.Execute(w, page)
if err != nil {
log.Errorf("error executing changelog template: %v", err)
diff --git a/src/cmd/changelog-webapp/static/templates/changelog.html b/src/cmd/changelog-webapp/static/templates/changelog.html
index aeca7c9..638184e 100644
--- a/src/cmd/changelog-webapp/static/templates/changelog.html
+++ b/src/cmd/changelog-webapp/static/templates/changelog.html
@@ -18,7 +18,7 @@
</div>
<div class="main">
<h1>Changelog</h1>
- <p class="feature-info">Retrieve a list of commits between two
+ <p class="feature-info">Retrieve a list of commits and sysctl changes between two
Container-Optimized OS builds.<br>
Example Input: <b>cos-rc-85-13310-1034-0</b> or <b>13310.1034.0</b>
</p>
@@ -43,8 +43,36 @@
<input type="text" class="target" name="target" placeholder="Image Name or Build Number" required>
{{end}}
<input type="hidden" name="n" value={{.QuerySize}}>
+ </div>
+ <div class="text">
+ <label>From </label>
+ {{if (ne .SourceMilestone "")}}
+ <input type="text" class="source" name="source milestone" placeholder="Milestone Number" value={{.SourceMilestone}} required>
+ {{else}}
+ <input type="text" class="source" name="source milestone" placeholder="Milestone Number" required>
+ {{end}}
+ <label> to </label>
+ {{if (ne .TargetMilestone "")}}
+ <input type="text" class="target" name="target milestone" placeholder="Milestone Number" value={{.TargetMilestone}} required>
+ {{else}}
+ <input type="text" class="target" name="target milestone" placeholder="Milestone Number" required>
+ {{end}}
<input class="submit" type="submit" value="Submit"><br>
</div>
+ <div class="text" style="display: none;">
+ <label>From </label>
+ {{if (ne .SourceBoard "")}}
+ <input type="hidden" class="source" name="source board" placeholder="Board Name" value={{.SourceBoard}} required>
+ {{else}}
+ <input type="hidden" class="source" name="source board" placeholder="Board Name" required>
+ {{end}}
+ <label> to </label>
+ {{if (ne .TargetBoard "")}}
+ <input type="hidden" class="target" name="target board" placeholder="Board Name" value={{.TargetBoard}} required>
+ {{else}}
+ <input type="hidden" class="target" name="target board" placeholder="Board Name" required>
+ {{end}}
+ </div>
<div class="radio">
{{if .Internal}}
<label>
@@ -146,6 +174,31 @@
</a>
{{end}}
{{end}}
+ {{if .Sysctl.NotEmpty}}
+ <h2>Runtime Sysctl Changes:</h2>
+ <div>{{.Sysctl.NotFound}}</div>
+ <table class="repo-table">
+ {{range $sysctl := .Sysctl.Added}}
+ <tr>
+ <td>Added:</td>
+ <td>{{$sysctl}}</td>
+ </tr>
+ {{end}}
+ {{range $sysctl := .Sysctl.Changed}}
+ <tr>
+ <td>Changed:</td>
+ <td>{{$sysctl}}</td>
+ </tr>
+ {{end}}
+ {{range $sysctl := .Sysctl.Deleted}}
+ <tr>
+ <td>Deleted:</td>
+ <td>{{$sysctl}}</td>
+ </tr>
+ {{end}}
+ </table>
+ {{end}}
+ </table>
</div>
</body>
</html>
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/static/templates/readme.html b/src/cmd/changelog-webapp/static/templates/readme.html
index 72a3c45..aef6415 100644
--- a/src/cmd/changelog-webapp/static/templates/readme.html
+++ b/src/cmd/changelog-webapp/static/templates/readme.html
@@ -62,6 +62,13 @@
go/crosland
</a>.
</p>
+ <p>
+ It also shows the sysctl changes between two builds if milestone
+ numbers are correctly provided and sysctl values have been dumped for these builds.
+ </p>
+ <p>
+ Eaxmple: Milestone Number = 89, Build Number = 16108.403.22
+ </p>
<h2>Find Build</h2>
<p>
This feature allows you to locate the first build containing a desired
diff --git a/src/pkg/changelog/changelog.go b/src/pkg/changelog/changelog.go
index 3d6fea3..81713e4 100644
--- a/src/pkg/changelog/changelog.go
+++ b/src/pkg/changelog/changelog.go
@@ -15,8 +15,9 @@
// This package generates a changelog based on the commit history between
// two build numbers. The changelog consists of two outputs - the commits
// added to the target build that aren't present in the source build, and the
-// commits in the source build that aren't present in the target build. This
-// package uses concurrency to improve performance.
+// commits in the source build that aren't present in the target build. It
+// also finds the sysctl value changes between two builds by fetch artifacts
+// from GCS. This package uses concurrency to improve performance.
//
// This package uses Gitiles to request information from a Git on Borg instance.
// To generate a changelog, the package first retrieves the the manifest files for
@@ -30,11 +31,16 @@
package changelog
import (
+ "context"
"errors"
+ "fmt"
+ "io/ioutil"
"net/http"
"regexp"
+ "sort"
"strings"
+ "cloud.google.com/go/storage"
"cos.googlesource.com/cos/tools.git/src/pkg/utils"
"github.com/beevik/etree"
@@ -298,7 +304,98 @@
}
}
outputChan <- additionsResult{Additions: repoCommits}
- return
+}
+
+// getSysctlDiff finds sysctl difference between the two builds.
+func GetSysctlDiff(bucket, pathSuffix, sourceBoard, sourceMilestone, source, targetBoard, targetMilestone, target string) (
+ []string, []string, []string, bool, bool) {
+ sourceBuildNum, targetBuildNum := resolveImageName(source), resolveImageName(target)
+ sourceChan := make(chan map[string]string)
+ targetChan := make(chan map[string]string)
+ go fetchSysctlToMap(fmt.Sprintf("%s/%s%s/R%s-%s",
+ bucket, sourceBoard, pathSuffix, sourceMilestone, sourceBuildNum), sourceChan)
+ go fetchSysctlToMap(fmt.Sprintf("%s/%s%s/R%s-%s",
+ bucket, targetBoard, pathSuffix, targetMilestone, targetBuildNum), targetChan)
+ sourceSysctl := <-sourceChan
+ targetSysctl := <-targetChan
+ foundSource := false
+ foundTarget := false
+ // if either one of the sysctl file doesn't exsit,
+ // return an empty list.
+ if len(sourceSysctl) > 0 {
+ foundSource = true
+ }
+ if len(targetSysctl) > 0 {
+ foundTarget = true
+ }
+ if !foundSource || !foundTarget {
+ return []string{}, []string{}, []string{}, foundSource, foundTarget
+ }
+
+ addList := []string{}
+ changeList := []string{}
+ deleteList := []string{}
+ for newName, newValue := range targetSysctl {
+ if oldValue, found := sourceSysctl[newName]; !found {
+ addList = append(addList,
+ fmt.Sprintf("%s: %s", newName, newValue))
+ } else if oldValue != newValue {
+ changeList = append(changeList,
+ fmt.Sprintf("%s: %s -> %s", newName, oldValue, newValue))
+ }
+ }
+ for oldName, oldValue := range sourceSysctl {
+ if _, found := targetSysctl[oldName]; !found {
+ deleteList = append(deleteList,
+ fmt.Sprintf("%s: %s", oldName, oldValue))
+ }
+ }
+
+ sort.Strings(addList)
+ sort.Strings(changeList)
+ sort.Strings(deleteList)
+
+ return addList, changeList, deleteList, foundSource, foundTarget
+}
+
+// fetchSysctlToMap fetches sysctl file from artifacts in GCS created
+// by build-executor and map each line to a <parameter_name: value>
+// pair.
+func fetchSysctlToMap(path string, outputChan chan map[string]string) {
+ outMap := make(map[string]string)
+ ctx := context.Background()
+ client, err := storage.NewClient(ctx)
+ if err != nil {
+ log.Errorf("failed to create storage client (error: %s)", err)
+ outputChan <- outMap
+ return
+ }
+
+ rc, err := client.Bucket(path).Object("sysctl_a.txt").NewReader(ctx)
+ if err != nil {
+ log.Errorf("failed to open %s at %s (error:%s)", "sysctl_a.txt", path, err)
+ outputChan <- outMap
+ return
+ }
+
+ byteBuf, err := ioutil.ReadAll(rc)
+ if err != nil {
+ log.Errorf("failed to read sysctl file (error:%s)", err)
+ outputChan <- outMap
+ return
+ }
+ separator := " = "
+ for _, line := range strings.Split(string(byteBuf), "\n") {
+ parts := strings.Split(line, separator)
+ // no value for this parameter
+ if len(parts) == 1 {
+ outMap[parts[0]] = " "
+ } else {
+ // assume the parameter name is before the first separator.
+ outMap[parts[0]] = strings.Join(parts[1:], separator)
+ }
+ }
+ outputChan <- outMap
}
// Changelog generates a changelog between 2 build numbers
@@ -324,6 +421,9 @@
//
// The second changelog contains all commits that are present in the source build
// but not present in the target build
+//
+// It also finds the sysctl value difference between two builds by comparing
+// sysctl files in GCS.
func Changelog(httpClient *http.Client, source, target, host, repo, croslandURL string, querySize int) (map[string]*RepoLog, map[string]*RepoLog, utils.ChangelogError) {
if httpClient == nil {
log.Error("httpClient is nil")
