Merge "cos_image_analyzer: Milestone 3 - Rootfs, Stateful, OS-Configs, and Partition-Structure difference"
diff --git a/go.mod b/go.mod
index 35e1b01..83152aa 100644
--- a/go.mod
+++ b/go.mod
@@ -8,6 +8,7 @@
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
github.com/google/go-cmp v0.5.1
github.com/google/subcommands v1.2.0
+ github.com/gorilla/sessions v1.2.0
github.com/julienschmidt/httprouter v1.3.0 // indirect
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.6.0
diff --git a/go.sum b/go.sum
index d6ed7da..a6e9e61 100644
--- a/go.sum
+++ b/go.sum
@@ -52,7 +52,6 @@
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -118,6 +117,10 @@
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
+github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
+github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ=
+github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
@@ -130,17 +133,13 @@
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -166,7 +165,6 @@
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
diff --git a/src/cmd/changelog-webapp/controllers/authHandlers.go b/src/cmd/changelog-webapp/controllers/authHandlers.go
new file mode 100644
index 0000000..56785cf
--- /dev/null
+++ b/src/cmd/changelog-webapp/controllers/authHandlers.go
@@ -0,0 +1,140 @@
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+ "math/rand"
+ "net/http"
+ "os"
+ "time"
+
+ "github.com/gorilla/sessions"
+ log "github.com/sirupsen/logrus"
+ "golang.org/x/oauth2"
+ "golang.org/x/oauth2/google"
+)
+
+const (
+ // Session variables
+ sessionName = "changelog"
+ sessionKeyLength = 32
+
+ // Oauth state generation variables
+ oauthStateCharset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890"
+ oauthStateLength = 16
+)
+
+var config = &oauth2.Config{
+ ClientID: os.Getenv("OAUTH_CLIENT_ID"),
+ ClientSecret: os.Getenv("OAUTH_CLIENT_SECRET"),
+ Endpoint: google.Endpoint,
+ RedirectURL: "https://cos-oss-interns-playground.uc.r.appspot.com/oauth2callback/",
+ Scopes: []string{"https://www.googleapis.com/auth/gerritcodereview"},
+}
+
+var store = sessions.NewCookieStore([]byte(randomString(sessionKeyLength)))
+
+func randomString(stringSize int) string {
+ randWithSeed := rand.New(rand.NewSource(time.Now().UnixNano()))
+ stateArr := make([]byte, stringSize)
+ for i := range stateArr {
+ stateArr[i] = oauthStateCharset[randWithSeed.Intn(len(oauthStateCharset))]
+ }
+ return string(stateArr)
+}
+
+// HTTPClient creates an authorized HTTP Client
+func HTTPClient(r *http.Request) (*http.Client, error) {
+ var parsedExpiry time.Time
+ session, err := store.Get(r, sessionName)
+ if err != nil {
+ return nil, fmt.Errorf("HTTPClient: No session found with sessionName %s", sessionName)
+ }
+ for _, key := range []string{"accessToken", "refreshToken", "tokenType", "expiry"} {
+ if val, ok := session.Values[key]; !ok || val == nil {
+ return nil, fmt.Errorf("HTTPClient: Session missing key %s", key)
+ }
+ }
+ if parsedExpiry, err = time.Parse(time.RFC3339, session.Values["expiry"].(string)); err != nil {
+ return nil, fmt.Errorf("HTTPClient: Token expiry is in an incorrect format")
+ }
+ token := &oauth2.Token{
+ AccessToken: session.Values["accessToken"].(string),
+ RefreshToken: session.Values["refreshToken"].(string),
+ TokenType: session.Values["tokenType"].(string),
+ Expiry: parsedExpiry,
+ }
+ return config.Client(context.Background(), token), nil
+}
+
+// HandleLogin handles login
+func HandleLogin(w http.ResponseWriter, r *http.Request) {
+ state := randomString(oauthStateLength)
+ session, _ := store.Get(r, sessionName)
+ session.Values["oauthState"] = state
+ err := session.Save(r, w)
+ if err != nil {
+ log.Errorf("HandleLogin: Error saving key: %v", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ authURL := config.AuthCodeURL(state, oauth2.AccessTypeOffline)
+ http.Redirect(w, r, authURL, http.StatusTemporaryRedirect)
+}
+
+// HandleCallback handles callback
+func HandleCallback(w http.ResponseWriter, r *http.Request) {
+ if err := r.ParseForm(); err != nil {
+ log.Errorf("Could not parse query: %v\n", err)
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ authCode := r.FormValue("code")
+ callbackState := r.FormValue("state")
+
+ session, err := store.Get(r, sessionName)
+ if err != nil {
+ log.Errorf("HandleCallback: Error retrieving session: %v", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if callbackState != session.Values["oauthState"].(string) {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ token, err := config.Exchange(context.Background(), authCode)
+ if err != nil {
+ log.Errorf("HandleCallback: Error exchanging token: %v", token)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ session.Values["accessToken"] = token.AccessToken
+ session.Values["refreshToken"] = token.RefreshToken
+ session.Values["tokenType"] = token.TokenType
+ session.Values["expiry"] = token.Expiry.Format(time.RFC3339)
+ err = session.Save(r, w)
+ if err != nil {
+ log.Errorf("HandleCallback: Error saving session: %v", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ http.Redirect(w, r, "/", http.StatusPermanentRedirect)
+}
diff --git a/src/cmd/changelog-webapp/controllers/pageHandlers.go b/src/cmd/changelog-webapp/controllers/pageHandlers.go
new file mode 100644
index 0000000..2304be7
--- /dev/null
+++ b/src/cmd/changelog-webapp/controllers/pageHandlers.go
@@ -0,0 +1,241 @@
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controllers
+
+import (
+ "fmt"
+ "net/http"
+ "os"
+ "strconv"
+ "strings"
+ "text/template"
+
+ "cos.googlesource.com/cos/tools/src/pkg/changelog"
+
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ subjectLen int = 100
+)
+
+var (
+ internalInstance string
+ internalManifestRepo string
+ externalInstance string
+ externalManifestRepo string
+ envQuerySize string
+ staticBasePath string
+ indexTemplate *template.Template
+ changelogTemplate *template.Template
+ promptLoginTemplate *template.Template
+)
+
+func init() {
+ internalInstance = os.Getenv("COS_INTERNAL_INSTANCE")
+ internalManifestRepo = os.Getenv("COS_INTERNAL_MANIFEST_REPO")
+ externalInstance = os.Getenv("COS_EXTERNAL_INSTANCE")
+ externalManifestRepo = os.Getenv("COS_EXTERNAL_MANIFEST_REPO")
+ envQuerySize = getIntVerifiedEnv("CHANGELOG_QUERY_SIZE")
+ staticBasePath = os.Getenv("STATIC_BASE_PATH")
+ indexTemplate = template.Must(template.ParseFiles(staticBasePath + "templates/index.html"))
+ changelogTemplate = template.Must(template.ParseFiles(staticBasePath + "templates/changelog.html"))
+ promptLoginTemplate = template.Must(template.ParseFiles(staticBasePath + "templates/promptLogin.html"))
+}
+
+type changelogData struct {
+ Instance string
+ Source string
+ Target string
+ Additions map[string]*changelog.RepoLog
+ Removals map[string]*changelog.RepoLog
+ Internal bool
+}
+
+type changelogPage struct {
+ Source string
+ Target string
+ QuerySize string
+ RepoTables []*repoTable
+ Internal bool
+}
+
+type repoTable struct {
+ Name string
+ Additions []*repoTableEntry
+ Removals []*repoTableEntry
+ AdditionsLink string
+ RemovalsLink string
+}
+
+type repoTableEntry struct {
+ IsAddition bool
+ SHA *shaAttr
+ Subject string
+ Bugs []*bugAttr
+ AuthorName string
+ CommitterName string
+ CommitTime string
+ ReleaseNote string
+}
+
+type shaAttr struct {
+ Name string
+ URL string
+}
+
+type bugAttr struct {
+ Name string
+ URL string
+}
+
+type promptLoginPage struct {
+ ActivePage string
+}
+
+// getIntVerifiedEnv retrieves an environment variable but checks that it can be
+// converted to int first
+func getIntVerifiedEnv(envName string) string {
+ output := os.Getenv(envName)
+ if _, err := strconv.Atoi(output); err != nil {
+ log.Errorf("getEnvAsInt: Failed to parse env variable %s with value %s: %v",
+ envName, os.Getenv(output), err)
+ }
+ return output
+}
+
+func gobCommitLink(instance, repo, SHA string) string {
+ return fmt.Sprintf("https://%s/%s/+/%s", instance, repo, SHA)
+}
+
+func gobDiffLink(instance, repo, sourceSHA, targetSHA string) string {
+ return fmt.Sprintf("https://%s/%s/+log/%s..%s?n=10000", instance, repo, sourceSHA, targetSHA)
+}
+
+func createRepoTableEntry(instance string, repo string, commit *changelog.Commit, isAddition bool) *repoTableEntry {
+ entry := new(repoTableEntry)
+ entry.IsAddition = isAddition
+ entry.SHA = &shaAttr{Name: commit.SHA[:8], URL: gobCommitLink(instance, repo, commit.SHA)}
+ entry.Subject = commit.Subject
+ if len(entry.Subject) > subjectLen {
+ entry.Subject = entry.Subject[:subjectLen]
+ }
+ entry.Bugs = make([]*bugAttr, len(commit.Bugs))
+ for i, bugURL := range commit.Bugs {
+ name := bugURL[strings.Index(bugURL, "/")+1:]
+ entry.Bugs[i] = &bugAttr{Name: name, URL: "http://" + bugURL}
+ }
+ entry.AuthorName = commit.AuthorName
+ entry.CommitterName = commit.CommitterName
+ entry.CommitTime = commit.CommitTime
+ entry.ReleaseNote = commit.ReleaseNote
+ return entry
+}
+
+func createChangelogPage(data changelogData) *changelogPage {
+ page := &changelogPage{Source: data.Source, Target: data.Target, QuerySize: envQuerySize, Internal: data.Internal}
+ for repoName, repoLog := range data.Additions {
+ table := &repoTable{Name: repoName}
+ for _, commit := range repoLog.Commits {
+ tableEntry := createRepoTableEntry(data.Instance, repoName, commit, true)
+ table.Additions = append(table.Additions, tableEntry)
+ }
+ if _, ok := data.Removals[repoName]; ok {
+ for _, commit := range data.Removals[repoName].Commits {
+ tableEntry := createRepoTableEntry(data.Instance, repoName, commit, false)
+ table.Removals = append(table.Removals, tableEntry)
+ }
+ if data.Removals[repoName].HasMoreCommits {
+ table.RemovalsLink = gobDiffLink(data.Instance, repoName, repoLog.TargetSHA, repoLog.SourceSHA)
+ }
+ }
+ if repoLog.HasMoreCommits {
+ table.AdditionsLink = gobDiffLink(data.Instance, repoName, repoLog.SourceSHA, repoLog.TargetSHA)
+ }
+ page.RepoTables = append(page.RepoTables, table)
+ }
+ // Add remaining repos that had removals but no additions
+ for repoName, repoLog := range data.Removals {
+ if _, ok := data.Additions[repoName]; ok {
+ continue
+ }
+ table := &repoTable{Name: repoName}
+ for _, commit := range repoLog.Commits {
+ tableEntry := createRepoTableEntry(data.Instance, repoName, commit, false)
+ table.Removals = append(table.Removals, tableEntry)
+ }
+ page.RepoTables = append(page.RepoTables, table)
+ if repoLog.HasMoreCommits {
+ table.RemovalsLink = gobDiffLink(data.Instance, repoName, repoLog.TargetSHA, repoLog.SourceSHA)
+ }
+ }
+ return page
+}
+
+// HandleIndex serves the home page
+func HandleIndex(w http.ResponseWriter, r *http.Request) {
+ indexTemplate.Execute(w, nil)
+}
+
+// HandleChangelog serves the changelog page
+func HandleChangelog(w http.ResponseWriter, r *http.Request) {
+ if err := r.ParseForm(); err != nil {
+ changelogTemplate.Execute(w, &changelogPage{QuerySize: envQuerySize})
+ return
+ }
+ httpClient, err := HTTPClient(r)
+ if err != nil {
+ log.Debug(err)
+ err = promptLoginTemplate.Execute(w, &promptLoginPage{ActivePage: "changelog"})
+ if err != nil {
+ log.Errorf("HandleChangelog: error executing promptLogin template: %v", err)
+ }
+ return
+ }
+ source := r.FormValue("source")
+ target := r.FormValue("target")
+ // If no source/target values specified in request, display empty changelog page
+ if source == "" || target == "" {
+ changelogTemplate.Execute(w, &changelogPage{QuerySize: envQuerySize, Internal: true})
+ return
+ }
+ querySize, err := strconv.Atoi(r.FormValue("n"))
+ if err != nil {
+ querySize, _ = strconv.Atoi(envQuerySize)
+ }
+ internal, instance, manifestRepo := false, externalInstance, externalManifestRepo
+ if r.FormValue("internal") == "true" {
+ internal, instance, manifestRepo = true, internalInstance, internalManifestRepo
+ }
+ added, removed, err := changelog.Changelog(httpClient, source, target, instance, manifestRepo, querySize)
+ if err != nil {
+ log.Errorf("HandleChangelog: error retrieving changelog between builds %s and %s on GoB instance: %s with manifest repository: %s\n%v\n",
+ source, target, externalInstance, externalManifestRepo, err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ page := createChangelogPage(changelogData{
+ Instance: instance,
+ Source: source,
+ Target: target,
+ Additions: added,
+ Removals: removed,
+ Internal: internal,
+ })
+ err = changelogTemplate.Execute(w, page)
+ if err != nil {
+ log.Errorf("HandleChangelog: error executing changelog template: %v", err)
+ }
+}
diff --git a/src/cmd/changelog-webapp/main.go b/src/cmd/changelog-webapp/main.go
new file mode 100644
index 0000000..fb95de8
--- /dev/null
+++ b/src/cmd/changelog-webapp/main.go
@@ -0,0 +1,54 @@
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "net/http"
+ "os"
+
+ "cos.googlesource.com/cos/tools/src/cmd/changelog-webapp/controllers"
+
+ log "github.com/sirupsen/logrus"
+)
+
+var (
+ staticBasePath string
+ port string
+)
+
+func init() {
+ staticBasePath = os.Getenv("STATIC_BASE_PATH")
+ port = os.Getenv("PORT")
+}
+
+func main() {
+ log.SetLevel(log.DebugLevel)
+
+ http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(staticBasePath))))
+ http.HandleFunc("/", controllers.HandleIndex)
+ http.HandleFunc("/login/", controllers.HandleLogin)
+ http.HandleFunc("/changelog/", controllers.HandleChangelog)
+ http.HandleFunc("/oauth2callback/", controllers.HandleCallback)
+
+ if port == "" {
+ port = "8081"
+ log.Printf("Defaulting to port %s", port)
+ }
+
+ log.Printf("Listening on port %s", port)
+ if err := http.ListenAndServe(":"+port, nil); err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/src/cmd/changelog-webapp/static/css/base.css b/src/cmd/changelog-webapp/static/css/base.css
new file mode 100644
index 0000000..1180143
--- /dev/null
+++ b/src/cmd/changelog-webapp/static/css/base.css
@@ -0,0 +1,87 @@
+html,
+body {
+ background-color: #f8f8f8;
+ font-family: 'Roboto', 'Noto', sans-serif;
+ font-weight: 300;
+ margin: 0;
+ overflow-x: hidden;
+ padding: 0;
+}
+
+h1 {
+ font-size: 24;
+}
+
+h2 {
+ font-size: 18;
+}
+
+p,
+p a {
+ font-size: 16;
+}
+
+a:link,
+a:visited {
+ color: blue;
+ text-decoration: none;
+}
+
+label {
+ font-size: 16;
+}
+
+.navbar {
+ align-items: center;
+ background-color: #4285f4;
+ display: flex;
+ height: 50px;
+ padding: 0px 25px;
+ width: 100%;
+}
+
+.navbar-title {
+ color: #f8f8f8;
+ font-size: 24;
+ font-weight: 350;
+}
+
+.sidenav {
+ height: 100%;
+ left: 0;
+ margin-top: 50px;
+ overflow-x: hidden;
+ padding-top: 30px;
+ position: fixed;
+ top: 0;
+ width: 120px;
+ z-index: 1;
+}
+
+.sidenav a {
+ color: #272727;
+ display: block;
+ font-size: 16px;
+ padding: 0px 8px 8px 26px;
+ text-decoration: none;
+ transition: 0.2s;
+}
+
+.sidenav a.active {
+ color: #4285f4;
+ font-weight: bold;
+}
+
+.sidenav a:hover {
+ color: #3d3d3d;
+ font-weight: 550;
+}
+
+.main {
+ margin: 30px 25px 0px 155px;
+}
+
+.text-content {
+ margin: auto;
+ max-width: 700px;
+}
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/static/css/changelog.css b/src/cmd/changelog-webapp/static/css/changelog.css
new file mode 100644
index 0000000..4a7ba7e
--- /dev/null
+++ b/src/cmd/changelog-webapp/static/css/changelog.css
@@ -0,0 +1,110 @@
+th {
+ font-weight: 440;
+ text-align: left;
+ padding-bottom: 0px;
+}
+
+th,
+td {
+ margin-bottom: 5px;
+ padding: 2px;
+}
+
+.changelog-form .text input {
+ height: 24px;
+ margin-bottom: 8px;
+ width: 135px;
+}
+
+.changelog-form .text .submit {
+ height: 24px;
+ width: 59px;
+}
+
+.changelog-form .text label {
+ margin: 4px;
+}
+
+.changelog-form .radio .external {
+ margin-left: 20px;
+}
+
+.sha-legend {
+ margin-top: 10px;
+}
+
+.sha-legend .legend-row {
+ display: flex;
+ flex-flow: row nowrap;
+ font-size: 14px;
+ margin: 3px 0px 3px 4px;
+}
+
+.sha-legend .circle {
+ border-radius: 50%;
+ height: 14px;
+ margin-right: 8px;
+ width: 14px;
+}
+
+.sha-legend .circle.addition {
+ background-color: #adef97;
+}
+
+.sha-legend .circle.removal {
+ background-color: #ffc0c0;
+}
+
+.repo-header {
+ margin: 24px 0px 12px 0px;
+}
+
+.repo-table {
+ border-spacing: 2px;
+ font-size: 14;
+ font-weight: 300;
+ margin: 4px 0px;
+ table-layout: fixed;
+}
+
+.commit-sha {
+ text-align: center;
+ width: 65px;
+}
+
+th.commit-sha {
+ text-align: left;
+}
+
+.addition {
+ background-color: #c2ffae;
+}
+
+.removal {
+ background-color: #ffdada
+}
+
+.commit-subject {
+ width: 520px;
+}
+
+.commit-bugs {
+ width: 115px;
+}
+
+.commit-author {
+ width: 150px;
+}
+
+.commit-committer {
+ width: 150px;
+}
+
+.commit-time {
+ width: 120px;
+}
+
+.gob-link {
+ font-size: 14;
+ padding-left: 2px;
+}
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/static/templates/changelog.html b/src/cmd/changelog-webapp/static/templates/changelog.html
new file mode 100644
index 0000000..93a3ca0
--- /dev/null
+++ b/src/cmd/changelog-webapp/static/templates/changelog.html
@@ -0,0 +1,122 @@
+<html>
+<head>
+ <meta name="description" content="Google COS build changelog">
+ <link rel="stylesheet" href="/static/css/base.css">
+ <link rel="stylesheet" href="/static/css/changelog.css">
+</head>
+<body>
+ <div class="navbar">
+ <p class="navbar-title">Container Optimized OS</p>
+ </div>
+ <div class="sidenav">
+ <a href="/">Home</a>
+ <a class="active" href="/changelog/">Changelog</a>
+ <a href="/locatecl/">Locate CL</a>
+ <a href="/login/">Login</a>
+ </div>
+ <div class="main">
+ <h1>Search Changelog</h1>
+ <form class="changelog-form" action="/changelog">
+ <div class="text">
+ <label for="source">From </label>
+ {{if (ne .Source "")}}
+ <input type="text" class="source" name="source" placeholder="COS build number" value={{.Source}} autocomplete="off">
+ {{else}}
+ <input type="text" class="source" name="source" placeholder="COS build number" autocomplete="off">
+ {{end}}
+ <label for="target"> to </label>
+ {{if (ne .Target "")}}
+ <input type="text" class="target" name="target" placeholder="COS build number" value={{.Target}} autocomplete="off" required>
+ {{else}}
+ <input type="text" class="target" name="target" placeholder="COS build number" autocomplete="off" required>
+ {{end}}
+ <input type="hidden" name="n" value={{.QuerySize}}>
+ <input class="submit" type="submit" value="Submit"><br>
+ </div>
+ <div class="radio">
+ {{if .Internal}}
+ <input type="radio" class="internal" name="internal" value="true" checked>
+ <label for="internal"> Internal </label>
+ <input type="radio" class="external" name="internal" value="false">
+ <label for="external"> External </label>
+ {{else}}
+ <input type="radio" class="internal" name="internal" value="true">
+ <label for="internal"> Internal </label>
+ <input type="radio" class="external" name="internal" value="false" checked>
+ <label for="external"> External </label>
+ {{end}}
+ </div>
+ </form>
+ {{if (and (ne .Target "") (ne .Source ""))}}
+ <div class="sha-legend">
+ <div class="legend-row">
+ <div class="circle addition"></div>
+ <span>Commits introduced to build {{.Target}} since build {{.Source}}</span><br>
+ </div>
+ <div class="legend-row">
+ <div class="circle removal"></div>
+ <span>Commits introduced to build {{.Source}} since build {{.Target}}</span>
+ </div>
+ </div>
+ {{end}}
+ {{range $table := .RepoTables}}
+ <h2 class="repo-header"> {{$table.Name}} </h2>
+ <table class="repo-table">
+ <tr>
+ <th class="commit-sha">SHA</th>
+ <th class="commit-subject">Subject</th>
+ <th class="commit-bugs">Bugs</th>
+ <th class="commit-author">Author</th>
+ <th class="commit-committer">Committer</th>
+ <th class="commit-time">Date</th>
+ <th class="commit-release-notes">Release Notes</th>
+ </tr>
+ </table>
+ <table class="repo-table">
+ {{range $commit := $table.Additions}}
+ <tr>
+ <td class="commit-sha addition">
+ <a href={{$commit.SHA.URL}} target="_blank">{{$commit.SHA.Name}}</a>
+ </td>
+ <td class="commit-subject">{{$commit.Subject}}</td>
+ <td class="commit-bugs">
+ {{range $bugAttr := $commit.Bugs}}
+ <a href={{$bugAttr.URL}} target="_blank">{{$bugAttr.Name}}</a>
+ {{end}}
+ </td>
+ <td class="commit-author">{{$commit.AuthorName}}</td>
+ <td class="commit-committer">{{$commit.CommitterName}}</td>
+ <td class="commit-time">{{$commit.CommitTime}}</td>
+ <td class="commit-release-notes">{{$commit.ReleaseNote}}</td>
+ </tr>
+ {{end}}
+ </table>
+ {{if (ne $table.AdditionsLink "")}}
+ <a class="gob-link" href={{$table.AdditionsLink}} target="_blank">Show more commits</a>
+ {{end}}
+ <table class="repo-table">
+ {{range $commit := $table.Removals}}
+ <tr>
+ <td class="commit-sha removal">
+ <a href={{$commit.SHA.URL}} target="_blank">{{$commit.SHA.Name}}</a>
+ </td>
+ <td class="commit-subject">{{$commit.Subject}}</td>
+ <td class="commit-bugs">
+ {{range $bugAttr := $commit.Bugs}}
+ <a href={{$bugAttr.URL}} target="_blank">{{$bugAttr.Name}}</a>
+ {{end}}
+ </td>
+ <td class="commit-author">{{$commit.AuthorName}}</td>
+ <td class="commit-committer">{{$commit.CommitterName}}</td>
+ <td class="commit-time">{{$commit.CommitTime}}</td>
+ <td class="commit-release-notes">{{$commit.ReleaseNote}}</td>
+ </tr>
+ {{end}}
+ </table>
+ {{if (ne $table.RemovalsLink "")}}
+ <a class="gob-link" href={{$table.RemovalsLink}} target="_blank">Show more commits</a>
+ {{end}}
+ {{end}}
+ </div>
+</body>
+</html>
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/static/templates/index.html b/src/cmd/changelog-webapp/static/templates/index.html
new file mode 100644
index 0000000..69e5e69
--- /dev/null
+++ b/src/cmd/changelog-webapp/static/templates/index.html
@@ -0,0 +1,30 @@
+<html>
+<head>
+ <meta name="description" content="Google COS build information">
+ <link rel="stylesheet" href="/static/css/base.css">
+</head>
+<body>
+ <div class="navbar">
+ <p class="navbar-title">Container Optimized OS</p>
+ </div>
+ <div class="sidenav">
+ <a class="active" href="/">Home</a>
+ <a href="/changelog/">Changelog</a>
+ <a href="/locatecl/">Locate CL</a>
+ <a href="/login/">Login</a>
+ </div>
+ <div class="main">
+ <div class="text-content">
+ <h1>Container-Optimized OS</h1>
+ <h2>A small, secure, stand alone VM image for building on top of Google Cloud</h2>
+ <p>Container-Optimized OS is an operating system image for your Compute Engine VMs that is optimized for
+ running
+ Docker containers. With Container-Optimized OS, you can bring up your Docker containers on Google Cloud
+ Platform
+ quickly, efficiently, and securely. Container-Optimized OS is maintained by Google and is based on the
+ open
+ source Chromium OS project.</p>
+ </div>
+ </div>
+</body>
+</html>
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/static/templates/promptLogin.html b/src/cmd/changelog-webapp/static/templates/promptLogin.html
new file mode 100644
index 0000000..6091473
--- /dev/null
+++ b/src/cmd/changelog-webapp/static/templates/promptLogin.html
@@ -0,0 +1,33 @@
+<html>
+<head>
+ <meta name="description" content="Google COS build information">
+ <link rel="stylesheet" href="/static/css/base.css">
+</head>
+<body>
+ <div class="navbar">
+ <p class="navbar-title">Container Optimized OS</p>
+ </div>
+ <div class="sidenav">
+ <a href="/">Home</a>
+ {{if (eq .ActivePage "changelog")}}
+ <a class="active" href="/changelog/">Changelog</a>
+ {{else}}
+ <a href="/changelog/">Changelog</a>
+ {{end}}
+ {{if (eq .ActivePage "locateCl")}}
+ <a class="active" href="/locatecl/">Locate CL</a>
+ {{else}}
+ <a href="/locatecl/">Locate CL</a>
+ {{end}}
+ <a href="/login/">Login</a>
+ </div>
+ <div class="main">
+ <div class="text-content">
+ <h1>Please sign in to use this feature</h1>
+ <p>This application requires OAuth authentication to communicate with Google Git repositories. Your account data
+ will not be used for any other purposes. To continue, please sign in by clicking <a href="/login/">here</a>.
+ </p>
+ </div>
+ </div>
+</body>
+</html>
\ No newline at end of file
diff --git a/src/cmd/changelog/main.go b/src/cmd/changelog/main.go
index 3eb0731..4531d36 100755
--- a/src/cmd/changelog/main.go
+++ b/src/cmd/changelog/main.go
@@ -57,7 +57,7 @@
return oauth2.NewClient(oauth2.NoContext, creds.TokenSource), nil
}
-func writeChangelogAsJSON(source string, target string, changes map[string][]*changelog.Commit) error {
+func writeChangelogAsJSON(source string, target string, changes map[string]*changelog.RepoLog) error {
fileName := fmt.Sprintf("%s -> %s.json", source, target)
log.Infof("Writing changelog to %s\n", fileName)
jsonData, err := json.MarshalIndent(changes, "", " ")
@@ -76,7 +76,7 @@
if err != nil {
return fmt.Errorf("generateChangelog: failed to create http client: \n%v", err)
}
- sourceToTargetChanges, targetToSourceChanges, err := changelog.Changelog(httpClient, source, target, instance, manifestRepo)
+ sourceToTargetChanges, targetToSourceChanges, err := changelog.Changelog(httpClient, source, target, instance, manifestRepo, -1)
if err != nil {
return fmt.Errorf("generateChangelog: error retrieving changelog between builds %s and %s on GoB instance: %s with manifest repository: %s\n%v",
source, target, instance, manifestRepo, err)
diff --git a/src/cmd/changelog/main_test.py b/src/cmd/changelog/main_test.py
index 251a9ad..9751fd3 100644
--- a/src/cmd/changelog/main_test.py
+++ b/src/cmd/changelog/main_test.py
@@ -62,8 +62,8 @@
data = json.load(f)
if len(data) == 0:
return False
- for repo, commit_log in data.items():
- for commit in commit_log:
+ for repoName, repoLog in data.items():
+ for commit in repoLog['Commits']:
if not check_commit_schema(commit):
return False
return True
@@ -71,10 +71,15 @@
class TestCLIApplication(unittest.TestCase):
- def test_build(self):
+ def setUp(self):
process = subprocess.run(["go", "build", "-o", "changelog","main.go"])
assert process.returncode == 0
+ def tearDown(self):
+ delete_logs("15000.0.0", "15055.0.0")
+ delete_logs("15050.0.0", "15056.0.0")
+ delete_logs("15056.0.0", "15056.0.0")
+
def test_basic_run(self):
source = "15050.0.0"
target = "15056.0.0"
diff --git a/src/cmd/cos_gpu_installer/internal/commands/install.go b/src/cmd/cos_gpu_installer/internal/commands/install.go
index 75a988c..2ef3611 100644
--- a/src/cmd/cos_gpu_installer/internal/commands/install.go
+++ b/src/cmd/cos_gpu_installer/internal/commands/install.go
@@ -3,6 +3,8 @@
import (
"context"
+ "fmt"
+ "io/ioutil"
"path/filepath"
"flag"
@@ -28,7 +30,7 @@
type InstallCommand struct {
driverVersion string
hostInstallDir string
- enforceSigning bool
+ unsignedDriver bool
internalDownload bool
debug bool
}
@@ -48,8 +50,10 @@
"The GPU driver verion to install. It will install the default GPU driver if the flag is not set explicitly.")
f.StringVar(&c.hostInstallDir, "dir", "/var/lib/nvidia",
"Host directory that GPU drivers should be installed to")
- f.BoolVar(&c.enforceSigning, "enforce-signing", true,
- "Whether to enforce GPU drivers being signed. Setting to false will disable kernel module signing security feature.")
+ f.BoolVar(&c.unsignedDriver, "allow-unsigned-driver", false,
+ "Whether to allow load unsigned GPU drivers. "+
+ "If this flag is set to true, module signing security features must be disabled on the host for driver installation to succeed. "+
+ "This flag is only for debugging.")
// TODO(mikewu): change this flag to a bucket prefix string.
f.BoolVar(&c.internalDownload, "internal-download", false,
"Whether to try to download files from Google internal server. This is only useful for internal developing.")
@@ -67,7 +71,7 @@
log.Infof("Running on COS build id %s", envReader.BuildNumber())
- downloader := &cos.GCSDownloader{envReader, c.internalDownload}
+ downloader := cos.NewGCSDownloader(envReader, c.internalDownload)
if c.driverVersion == "" {
defaultVersion, err := installer.GetDefaultGPUDriverVersion(downloader)
if err != nil {
@@ -78,19 +82,20 @@
}
log.Infof("Installing GPU driver version %s", c.driverVersion)
- if !c.enforceSigning {
- log.Info("Doesn't enforce signing. Need to disable module locking.")
- if err := cos.DisableKernelModuleLocking(); err != nil {
- c.logError(errors.Wrap(err, "failed to configure kernel module locking"))
- return subcommands.ExitFailure
+ if c.unsignedDriver {
+ kernelCmdline, err := ioutil.ReadFile("/proc/cmdline")
+ if err != nil {
+ c.logError(fmt.Errorf("failed to read kernel command line: %v", err))
+ }
+ if !cos.CheckKernelModuleSigning(string(kernelCmdline)) {
+ log.Warning("Current kernel command line does not support unsigned kernel modules. Not enforcing kernel module signing may cause installation fail.")
}
}
-
hostInstallDir := filepath.Join(hostRootPath, c.hostInstallDir)
cacher := installer.NewCacher(hostInstallDir, envReader.BuildNumber(), c.driverVersion)
if isCached, err := cacher.IsCached(); isCached && err == nil {
log.Info("Found cached version, NOT building the drivers.")
- if err := installer.ConfigureCachedInstalltion(hostInstallDir, c.enforceSigning); err != nil {
+ if err := installer.ConfigureCachedInstalltion(hostInstallDir, !c.unsignedDriver); err != nil {
c.logError(errors.Wrap(err, "failed to configure cached installation"))
return subcommands.ExitFailure
}
@@ -121,7 +126,7 @@
}
defer func() { callback <- 0 }()
- if c.enforceSigning {
+ if !c.unsignedDriver {
if err := signing.DownloadDriverSignatures(downloader, c.driverVersion); err != nil {
return errors.Wrap(err, "failed to download driver signature")
}
@@ -140,7 +145,7 @@
return errors.Wrap(err, "failed to install toolchain")
}
- if err := installer.RunDriverInstaller(installerFile, c.enforceSigning); err != nil {
+ if err := installer.RunDriverInstaller(installerFile, !c.unsignedDriver); err != nil {
return errors.Wrap(err, "failed to run GPU driver installer")
}
if err := cacher.Cache(); err != nil {
diff --git a/src/cmd/cos_gpu_installer/internal/commands/list.go b/src/cmd/cos_gpu_installer/internal/commands/list.go
index 63ec447..4b4e095 100644
--- a/src/cmd/cos_gpu_installer/internal/commands/list.go
+++ b/src/cmd/cos_gpu_installer/internal/commands/list.go
@@ -47,7 +47,7 @@
return subcommands.ExitFailure
}
log.Infof("Running on COS build id %s", envReader.BuildNumber())
- downloader := &cos.GCSDownloader{envReader, c.internalDownload}
+ downloader := cos.NewGCSDownloader(envReader, c.internalDownload)
artifacts, err := downloader.ListExtensionArtifacts("gpu")
if err != nil {
c.logError(errors.Wrap(err, "failed to list gpu extension artifacts"))
diff --git a/src/cmd/cos_gpu_installer/internal/installer/installer.go b/src/cmd/cos_gpu_installer/internal/installer/installer.go
index 0b8d6b4..654240c 100644
--- a/src/cmd/cos_gpu_installer/internal/installer/installer.go
+++ b/src/cmd/cos_gpu_installer/internal/installer/installer.go
@@ -22,7 +22,7 @@
const (
gpuInstallDirContainer = "/usr/local/nvidia"
- defaultGPUDriverFile = "default_version"
+ defaultGPUDriverFile = "gpu_default_version"
precompiledInstallerURLFormat = "https://storage.googleapis.com/nvidia-drivers-%s-public/nvidia-cos-project/%s/tesla/%s_00/%s/NVIDIA-Linux-x86_64-%s_%s-%s.cos"
defaultFilePermission = 0755
)
@@ -140,9 +140,8 @@
if needSigned {
// Run installer to compile drivers. Expect the command to fail as the drivers are not signed yet.
- if err := utils.RunCommandAndLogOutput(cmd, true); err != nil {
- return errors.Wrap(err, "failed to run GPU driver installer")
- }
+ utils.RunCommandAndLogOutput(cmd, true)
+
// sign GPU drivers.
kernelFiles, err := ioutil.ReadDir(filepath.Join(extractDir, "kernel"))
if err != nil {
@@ -184,9 +183,9 @@
}
// GetDefaultGPUDriverVersion gets the default GPU driver version.
-func GetDefaultGPUDriverVersion(downloader cos.ExtensionsDownloader) (string, error) {
+func GetDefaultGPUDriverVersion(downloader cos.ArtifactsDownloader) (string, error) {
log.Info("Getting the default GPU driver version")
- content, err := downloader.GetExtensionArtifact(cos.GpuExtension, defaultGPUDriverFile)
+ content, err := downloader.GetArtifact(defaultGPUDriverFile)
if err != nil {
return "", errors.Wrapf(err, "failed to get default GPU driver version")
}
diff --git a/src/pkg/changelog/changelog.go b/src/pkg/changelog/changelog.go
old mode 100755
new mode 100644
index e984581..3b6c686
--- a/src/pkg/changelog/changelog.go
+++ b/src/pkg/changelog/changelog.go
@@ -47,9 +47,9 @@
manifestFileName string = "snapshot.xml"
// These constants are used for exponential increase in Gitiles request size.
- defaultPageSize int32 = 1000
- pageSizeGrowthMultiplier int32 = 5
- maxPageSize int32 = 10000
+ defaultPageSize int = 1000
+ pageSizeGrowthMultiplier int = 5
+ maxPageSize int = 10000
)
type repo struct {
@@ -66,16 +66,26 @@
}
type commitsResult struct {
- RepoURL string
- Commits []*Commit
- Err error
+ RepoURL string
+ Commits []*Commit
+ HasMoreCommits bool
+ Err error
}
type additionsResult struct {
- Additions map[string][]*Commit
+ Additions map[string]*RepoLog
Err error
}
+// limitPageSize will restrict a request page size to min of pageSize (which grows exponentially)
+// or remaining request size
+func limitPageSize(pageSize, requestedSize int) int {
+ if requestedSize == -1 || pageSize <= requestedSize {
+ return pageSize
+ }
+ return requestedSize
+}
+
func gerritClient(httpClient *http.Client, remoteURL string) (gitilesProto.GitilesClient, error) {
log.Debugf("Creating Gerrit client for remote url %s\n", remoteURL)
cl, err := gitilesApi.NewRESTClient(httpClient, remoteURL, true)
@@ -124,7 +134,9 @@
// Extract the "name", "remote", and "revision" attributes from each project tag.
// Some projects do not have a "remote" attribute.
// If this is the case, they should use the default remoteURL.
- remoteMap[""] = remoteMap[root.SelectElement("default").SelectAttr("remote").Value]
+ if root.SelectElement("default").SelectAttr("remote") != nil {
+ remoteMap[""] = remoteMap[root.SelectElement("default").SelectAttr("remote").Value]
+ }
repos := make(map[string]*repo)
for _, project := range root.SelectElements("project") {
repos[project.SelectAttr("name").Value] = &repo{
@@ -158,15 +170,17 @@
}
// commits get all commits that occur between committish and ancestor for a specific repo.
-func commits(client gitilesProto.GitilesClient, repo string, committish string, ancestor string, outputChan chan commitsResult) {
+func commits(client gitilesProto.GitilesClient, repo string, committish string, ancestor string, querySize int, outputChan chan commitsResult) {
log.Debugf("Fetching changelog for repo: %s on committish %s\n", repo, committish)
start := time.Now()
- pageSize := defaultPageSize
+
+ pageSize := limitPageSize(defaultPageSize, querySize)
+ querySize -= pageSize
request := gitilesProto.LogRequest{
Project: repo,
Committish: committish,
ExcludeAncestorsOf: ancestor,
- PageSize: pageSize,
+ PageSize: int32(pageSize),
}
response, err := client.Log(context.Background(), &request)
if err != nil {
@@ -174,6 +188,7 @@
repo, committish, ancestor, err)}
return
}
+
// No nextPageToken means there were less than <defaultPageSize> commits total.
// We can immediately return.
if response.NextPageToken == "" {
@@ -184,21 +199,23 @@
repo, committish, ancestor, err)}
return
}
- outputChan <- commitsResult{RepoURL: repo, Commits: parsedCommits}
+ outputChan <- commitsResult{RepoURL: repo, Commits: parsedCommits, HasMoreCommits: (response.NextPageToken != "")}
return
}
// Retrieve remaining commits using exponential increase in pageSize.
allCommits := response.Log
- for response.NextPageToken != "" {
+ for querySize > 0 && response.NextPageToken != "" {
if pageSize < maxPageSize {
pageSize *= pageSizeGrowthMultiplier
}
+ pageSize = limitPageSize(pageSize, querySize)
+ querySize -= pageSize
request := gitilesProto.LogRequest{
Project: repo,
Committish: committish,
ExcludeAncestorsOf: ancestor,
PageToken: response.NextPageToken,
- PageSize: pageSize,
+ PageSize: int32(pageSize),
}
response, err = client.Log(context.Background(), &request)
if err != nil {
@@ -215,14 +232,14 @@
repo, committish, ancestor, err)}
return
}
- outputChan <- commitsResult{RepoURL: repo, Commits: parsedCommits}
+ outputChan <- commitsResult{RepoURL: repo, Commits: parsedCommits, HasMoreCommits: (response.NextPageToken != "")}
}
// additions retrieves all commits that occured between 2 parsed manifest files for each repo.
// Returns a map of repo name -> list of commits.
-func additions(clients map[string]gitilesProto.GitilesClient, sourceRepos map[string]*repo, targetRepos map[string]*repo, outputChan chan additionsResult) {
+func additions(clients map[string]gitilesProto.GitilesClient, sourceRepos map[string]*repo, targetRepos map[string]*repo, querySize int, outputChan chan additionsResult) {
log.Debug("Retrieving commit additions")
- repoCommits := make(map[string][]*Commit)
+ repoCommits := make(map[string]*RepoLog)
commitsChan := make(chan commitsResult, len(targetRepos))
for repoURL, targetRepoInfo := range targetRepos {
cl := clients[targetRepoInfo.InstanceURL]
@@ -232,7 +249,7 @@
if sourceRepoInfo, ok := sourceRepos[repoURL]; ok {
ancestorCommittish = sourceRepoInfo.Committish
}
- go commits(cl, repoURL, targetRepoInfo.Committish, ancestorCommittish, commitsChan)
+ go commits(cl, repoURL, targetRepoInfo.Committish, ancestorCommittish, querySize, commitsChan)
}
for i := 0; i < len(targetRepos); i++ {
res := <-commitsChan
@@ -240,8 +257,17 @@
outputChan <- additionsResult{Err: res.Err}
return
}
+ sourceSHA := ""
+ if sha, ok := sourceRepos[res.RepoURL]; ok {
+ sourceSHA = sha.Committish
+ }
if len(res.Commits) > 0 {
- repoCommits[res.RepoURL] = res.Commits
+ repoCommits[res.RepoURL] = &RepoLog{
+ Commits: res.Commits,
+ HasMoreCommits: res.HasMoreCommits,
+ SourceSHA: sourceSHA,
+ TargetSHA: targetRepos[res.RepoURL].Committish,
+ }
}
}
outputChan <- additionsResult{Additions: repoCommits}
@@ -257,19 +283,26 @@
// a tag that links directly to snapshot.xml
// Ex. For /refs/tags/15049.0.0, the argument should be 15049.0.0
//
-// The host should be the GoB instance that Manifest files are hosted in
+// host should be the GoB instance that Manifest files are hosted in
// ex. "cos.googlesource.com"
//
-// The repo should be the repository that build manifest files
+// repo should be the repository that build manifest files
// are located, ex. "cos/manifest-snapshots"
//
+// querySize should be the number of commits that should be included in each
+// repository changelog. Specify as -1 to get all commits
+//
// Outputs two changelogs
// The first changelog contains new commits that were added to the target
// build starting from the source build number
//
// The second changelog contains all commits that are present in the source build
// but not present in the target build
-func Changelog(httpClient *http.Client, sourceBuildNum string, targetBuildNum string, host string, repo string) (map[string][]*Commit, map[string][]*Commit, error) {
+func Changelog(httpClient *http.Client, sourceBuildNum string, targetBuildNum string, host string, repo string, querySize int) (map[string]*RepoLog, map[string]*RepoLog, error) {
+ if httpClient == nil {
+ return nil, nil, errors.New("Changelog: httpClient should not be nil")
+ }
+
log.Infof("Retrieving changelog between %s and %s\n", sourceBuildNum, targetBuildNum)
clients := make(map[string]gitilesProto.GitilesClient)
@@ -302,16 +335,16 @@
addChan := make(chan additionsResult, 1)
missChan := make(chan additionsResult, 1)
- go additions(clients, sourceRepos, targetRepos, addChan)
- go additions(clients, targetRepos, sourceRepos, missChan)
- addRes := <-addChan
- if addRes.Err != nil {
- return nil, nil, fmt.Errorf("Changelog: failure when retrieving commit additions:\n%v", err)
- }
+ go additions(clients, sourceRepos, targetRepos, querySize, addChan)
+ go additions(clients, targetRepos, sourceRepos, querySize, missChan)
missRes := <-missChan
if missRes.Err != nil {
return nil, nil, fmt.Errorf("Changelog: failure when retrieving missed commits:\n%v", err)
}
+ addRes := <-addChan
+ if addRes.Err != nil {
+ return nil, nil, fmt.Errorf("Changelog: failure when retrieving commit additions:\n%v", err)
+ }
return addRes.Additions, missRes.Additions, nil
}
diff --git a/src/pkg/changelog/changelog_test.go b/src/pkg/changelog/changelog_test.go
index 6113444..86b6423 100644
--- a/src/pkg/changelog/changelog_test.go
+++ b/src/pkg/changelog/changelog_test.go
@@ -51,9 +51,9 @@
return true
}
-func mappingInLog(log map[string][]*Commit, check []string) bool {
+func repoListInLog(log map[string]*RepoLog, check []string) bool {
for _, check := range check {
- if log, ok := log[check]; !ok || len(log) == 0 {
+ if log, ok := log[check]; !ok || len(log.Commits) == 0 {
return false
}
}
@@ -64,57 +64,68 @@
httpClient, err := getHTTPClient()
// Test invalid source
- additions, misses, err := Changelog(httpClient, "15", "15043.0.0", cosInstance, defaultManifestRepo)
+ additions, removals, err := Changelog(httpClient, "15", "15043.0.0", cosInstance, defaultManifestRepo, -1)
if additions != nil {
t.Errorf("Changelog failed, expected nil additions, got %v", additions)
- } else if misses != nil {
- t.Errorf("Changelog failed, expected nil misses, got %v", misses)
+ } else if removals != nil {
+ t.Errorf("Changelog failed, expected nil removals, got %v", removals)
} else if err == nil {
t.Errorf("Changelog failed, expected error, got nil")
}
// Test invalid target
- additions, misses, err = Changelog(httpClient, "15043.0.0", "abx", cosInstance, defaultManifestRepo)
+ additions, removals, err = Changelog(httpClient, "15043.0.0", "abx", cosInstance, defaultManifestRepo, -1)
if additions != nil {
t.Errorf("Changelog failed, expected nil additions, got %v", additions)
- } else if misses != nil {
- t.Errorf("Changelog failed, expected nil misses, got %v", misses)
+ } else if removals != nil {
+ t.Errorf("Changelog failed, expected nil removals, got %v", removals)
} else if err == nil {
t.Errorf("Changelog failed, expected error, got nil")
}
// Test invalid instance
- additions, misses, err = Changelog(httpClient, "15036.0.0", "15041.0.0", "com", defaultManifestRepo)
+ additions, removals, err = Changelog(httpClient, "15036.0.0", "15041.0.0", "com", defaultManifestRepo, -1)
if additions != nil {
t.Errorf("Changelog failed, expected nil additions, got %v", additions)
- } else if misses != nil {
- t.Errorf("Changelog failed, expected nil misses, got %v", misses)
+ } else if removals != nil {
+ t.Errorf("Changelog failed, expected nil removals, got %v", removals)
} else if err == nil {
t.Errorf("Changelog failed, expected error, got nil")
}
// Test invalid manifest repo
- additions, misses, err = Changelog(httpClient, "15036.0.0", "15041.0.0", cosInstance, "cos/not-a-repo")
+ additions, removals, err = Changelog(httpClient, "15036.0.0", "15041.0.0", cosInstance, "cos/not-a-repo", -1)
if additions != nil {
t.Errorf("Changelog failed, expected nil additions, got %v", additions)
- } else if misses != nil {
- t.Errorf("Changelog failed, expected nil misses, got %v", misses)
+ } else if removals != nil {
+ t.Errorf("Changelog failed, expected nil removals, got %v", removals)
} else if err == nil {
t.Errorf("Changelog failed, expected error, got nil")
}
// Test build number higher than latest release
- additions, misses, err = Changelog(httpClient, "15036.0.0", "99999.0.0", cosInstance, defaultManifestRepo)
+ additions, removals, err = Changelog(httpClient, "15036.0.0", "99999.0.0", cosInstance, defaultManifestRepo, -1)
if additions != nil {
t.Errorf("Changelog failed, expected nil additions, got %v", additions)
- } else if misses != nil {
- t.Errorf("Changelog failed, expected nil misses, got %v", misses)
+ } else if removals != nil {
+ t.Errorf("Changelog failed, expected nil removals, got %v", removals)
} else if err == nil {
t.Errorf("Changelog failed, expected error, got nil")
}
+ // Test manifest with remote urls specified and no default URL
+ additions, removals, err = Changelog(httpClient, "1.0.0", "2.0.0", cosInstance, defaultManifestRepo, -1)
+ if additions == nil {
+ t.Errorf("Changelog failed, expected additions, got nil")
+ } else if removals == nil {
+ t.Errorf("Changelog failed, expected removals, got nil")
+ } else if err != nil {
+ t.Errorf("Changelog failed, expected no error, got %v", err)
+ }
+
// Test 1 build number difference with only 1 repo change between them
// Ensure that commits are correctly inserted in proper order
+ // Check that changelog metadata correctly populated
source := "15050.0.0"
target := "15051.0.0"
expectedCommits := []string{
@@ -227,23 +238,29 @@
"9bc12bb411f357188d008864f80dfba43210b9d8",
"bf0dd3757826b9bc9d7082f5f749ff7615d4bcb3",
}
- additions, misses, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo)
+ additions, removals, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo, -1)
if err != nil {
t.Errorf("Changelog failed, expected no error, got %v", err)
- } else if len(misses) != 0 {
- t.Errorf("Changelog failed, expected empty misses list, got %v", misses)
+ } else if len(removals) != 0 {
+ t.Errorf("Changelog failed, expected empty removals list, got %v", removals)
} else if len(additions) != 1 {
t.Errorf("Changelog failed, expected only 1 repo in additions, got %v", additions)
- } else if _, ok := additions["cos/overlays/board-overlays"]; !ok {
- t.Errorf("Changelog failed, expected \"cos/overlays/board-overlays\" in additions, got %v", additions)
- } else if changes, _ := additions["cos/overlays/board-overlays"]; len(changes) != 108 {
+ }
+ boardOverlayLog := additions["cos/overlays/board-overlays"]
+ if boardOverlayLog == nil {
+ t.Errorf("Changelog failed, expected cos/overlays/board-overlays in changelog, got nil")
+ } else if changes := boardOverlayLog.Commits; len(changes) != 108 {
t.Errorf("Changelog failed, expected 108 changes for \"cos/overlays/board-overlays\", got %d", len(changes))
- } else if !commitsMatch(additions["cos/overlays/board-overlays"], expectedCommits) {
+ } else if !commitsMatch(boardOverlayLog.Commits, expectedCommits) {
t.Errorf("Changelog failed, Changelog output does not match expected commits or is not sorted")
+ } else if boardOverlayLog.SourceSHA != "612ca5ef5455534127d008e08c65aa29a2fd97a5" {
+ t.Errorf("Changelog failed, expected SourceSHA \"612ca5ef5455534127d008e08c65aa29a2fd97a5\", got %s", boardOverlayLog.SourceSHA)
+ } else if boardOverlayLog.TargetSHA != "6201c49afe667c8fa7796608a4d7162bb3f7f4f4" {
+ t.Errorf("Changelog failed, expected SourceSHA \"6201c49afe667c8fa7796608a4d7162bb3f7f4f4\", got %s", boardOverlayLog.TargetSHA)
}
// Test build numbers further apart from each other with multiple repo differences
- // Also ensures that misses are correctly populated
+ // Also ensures that removals are correctly populated
source = "15020.0.0"
target = "15056.0.0"
additionRepos := []string{
@@ -266,12 +283,89 @@
"mirrors/cros/chromiumos/repohooks",
"mirrors/cros/chromiumos/overlays/portage-stable",
}
- additions, misses, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo)
+ additions, removals, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo, -1)
if err != nil {
t.Errorf("Changelog failed, expected no error, got %v", err)
- } else if _, ok := misses["third_party/kernel"]; len(misses) != 1 && ok {
- t.Errorf("Changelog failed, expected miss list containing only \"third_party/kernel\", got %v", misses)
- } else if !mappingInLog(additions, additionRepos) {
+ }
+ kernelLog := additions["third_party/kernel"]
+ if len(removals) != 1 || kernelLog == nil {
+ t.Errorf("Changelog failed, expected miss list containing only \"third_party/kernel\", got %v", removals)
+ } else if !repoListInLog(additions, additionRepos) {
t.Errorf("Changelog failed, additions repo output does not match expected repos %v", additionRepos)
}
+
+ // Test changelog returns correct output when given a querySize instead of -1
+ source = "15030.0.0"
+ target = "15050.0.0"
+ querySize := 50
+ additions, removals, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo, querySize)
+ if err != nil {
+ t.Errorf("Changelog failed, expected no error, got %v", err)
+ } else if additions == nil {
+ t.Errorf("Changelog failed, non-empty expected additions, got nil")
+ } else if removals == nil {
+ t.Errorf("Changelog failed, non-empty expected removals, got nil")
+ } else if _, ok := additions["third_party/kernel"]; !ok {
+ t.Errorf("Changelog failed, expected repo: third_party/kernel in additions")
+ }
+ for repoName, repoLog := range additions {
+ if repoLog.Commits == nil || len(repoLog.Commits) == 0 {
+ t.Errorf("Changelog failed, expected non-empty additions commits, got nil or empty commits")
+ }
+ if len(repoLog.Commits) > querySize {
+ t.Errorf("Changelog failed, expected %d commits for repo: %s, got: %d", querySize, repoName, len(repoLog.Commits))
+ } else if repoName == "third_party/kernel" && !repoLog.HasMoreCommits {
+ t.Errorf("Changelog failed, expected HasMoreCommits = True for repo: third_party/kernel, got False")
+ } else if repoLog.HasMoreCommits && len(repoLog.Commits) < querySize {
+ t.Errorf("Changelog failed, expected HasMoreCommits = False for repo: %s with %d commits, got True", repoName, len(repoLog.Commits))
+ }
+ }
+
+ // Test changelog handles manifest with non-matching repositories
+ source = "12871.1177.0"
+ target = "12871.1179.0"
+ additions, removals, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo, querySize)
+ if err != nil {
+ t.Errorf("Changelog failed, expected no error, got %v", err)
+ } else if len(removals) != 0 {
+ t.Errorf("Changelog failed, expected empty removals, got %v", removals)
+ } else if _, ok := additions["cos/cobble"]; !ok {
+ t.Errorf("Changelog failed, expected repo: third_party/kernel in additions")
+ }
+ for repoName, repoLog := range additions {
+ if repoLog.Commits == nil || len(repoLog.Commits) == 0 {
+ t.Errorf("Changelog failed, expected non-empty additions commits, got nil or empty commits")
+ } else if repoName == "cos/cobble" {
+ if repoLog.HasMoreCommits {
+ t.Errorf("Changelog failed, expected hasMoreCommits = false for repo: cos/cobble, got true")
+ } else if repoLog.SourceSHA != "" {
+ t.Errorf("Changelog failed, expected empty SourceSHA for cos/cobble, got %s", repoLog.SourceSHA)
+ } else if repoLog.TargetSHA != "4ab43f1f86b7099b8ad75cf9615ea1fa155bbd7d" {
+ t.Errorf("Changelog failed, expected TargetSHA: \"4ab43f1f86b7099b8ad75cf9615ea1fa155bbd7d\" for cos/cobble, got %s", repoLog.TargetSHA)
+ }
+ }
+ }
+ source = "12871.1179.0"
+ target = "12871.1177.0"
+ additions, removals, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo, querySize)
+ if err != nil {
+ t.Errorf("Changelog failed, expected no error, got %v", err)
+ } else if len(additions) != 0 {
+ t.Errorf("Changelog failed, expected empty additions, got %v", additions)
+ } else if _, ok := removals["cos/cobble"]; !ok {
+ t.Errorf("Changelog failed, expected repo: third_party/kernel in additions")
+ }
+ for repoName, repoLog := range removals {
+ if repoLog.Commits == nil || len(repoLog.Commits) == 0 {
+ t.Errorf("Changelog failed, expected non-empty additions commits, got nil or empty commits")
+ } else if repoName == "cos/cobble" {
+ if repoLog.HasMoreCommits {
+ t.Errorf("Changelog failed, expected hasMoreCommits = false for repo: cos/cobble, got true")
+ } else if repoLog.SourceSHA != "" {
+ t.Errorf("Changelog failed, expected empty SourceSHA for cos/cobble, got %s", repoLog.SourceSHA)
+ } else if repoLog.TargetSHA != "4ab43f1f86b7099b8ad75cf9615ea1fa155bbd7d" {
+ t.Errorf("Changelog failed, expected TargetSHA: \"4ab43f1f86b7099b8ad75cf9615ea1fa155bbd7d\" for cos/cobble, got %s", repoLog.TargetSHA)
+ }
+ }
+ }
}
diff --git a/src/pkg/changelog/gitcommit.go b/src/pkg/changelog/gitcommit.go
index e52b2c7..39d49f4 100755
--- a/src/pkg/changelog/gitcommit.go
+++ b/src/pkg/changelog/gitcommit.go
@@ -18,13 +18,22 @@
"errors"
"regexp"
"strings"
- "time"
"go.chromium.org/luci/common/proto/git"
)
-const bugLinePrefix string = "BUG="
-const releaseNoteLinePrefix string = "RELEASE_NOTE="
+const (
+ bugLinePrefix string = "BUG="
+ releaseNoteLinePrefix string = "RELEASE_NOTE="
+)
+
+// RepoLog contains a changelist for a particular repository
+type RepoLog struct {
+ Commits []*Commit
+ SourceSHA string
+ TargetSHA string
+ HasMoreCommits bool
+}
// Commit is a simplified struct of git.Commit
// Useful for interfaces
@@ -104,7 +113,7 @@
func commitTime(commit *git.Commit) string {
if commit.Committer != nil {
- return commit.Committer.Time.AsTime().Format(time.RFC1123)
+ return commit.Committer.Time.AsTime().Format("Mon, 2 Jan 2006")
}
return "None"
}
diff --git a/src/pkg/changelog/gitcommit_test.go b/src/pkg/changelog/gitcommit_test.go
index 268245d..7c623ac 100644
--- a/src/pkg/changelog/gitcommit_test.go
+++ b/src/pkg/changelog/gitcommit_test.go
@@ -29,7 +29,7 @@
parent string = "7645df3136c5b5e43eb1af182b0c67d78ca2d517"
authorName string = "Austin Yuan"
committerName string = "Boston Yuan"
- timeVal string = "Sat, 01 Feb 2020 08:15:00 UTC"
+ timeVal string = "Sat, 1 Feb 2020"
)
var authorTime time.Time
@@ -339,9 +339,9 @@
case !reflect.DeepEqual(commit.Bugs, test.Bugs[i]):
t.Errorf("exptected bugs %#v, got %#v", test.Bugs[i], commit.Bugs)
case commit.ReleaseNote != test.ReleaseNote[i]:
- t.Errorf("expected release note %s, got %s", test.ReleaseNote, commit.ReleaseNote)
+ t.Errorf("expected release note %s, got %s", test.ReleaseNote[i], commit.ReleaseNote)
case commit.CommitTime != test.CommitTime[i]:
- t.Errorf("expected commit time %s, got %s", test.CommitTime, commit.CommitTime)
+ t.Errorf("expected commit time %s, got %s", test.CommitTime[i], commit.CommitTime)
}
}
})
diff --git a/src/pkg/cos/artifacts.go b/src/pkg/cos/artifacts.go
index 616778e..bb2f0bb 100644
--- a/src/pkg/cos/artifacts.go
+++ b/src/pkg/cos/artifacts.go
@@ -42,6 +42,11 @@
Internal bool
}
+// NewGCSDownloader creates a GCSDownloader instance.
+func NewGCSDownloader(e *EnvReader, i bool) *GCSDownloader {
+ return &GCSDownloader{e, i}
+}
+
// DownloadKernelSrc downloads COS kernel sources to destination directory.
func (d *GCSDownloader) DownloadKernelSrc(destDir string) error {
return d.DownloadArtifact(destDir, kernelSrcArchive)
diff --git a/src/pkg/cos/cos.go b/src/pkg/cos/cos.go
index c431c00..c75f7c5 100644
--- a/src/pkg/cos/cos.go
+++ b/src/pkg/cos/cos.go
@@ -8,7 +8,6 @@
"os/exec"
"path/filepath"
"strings"
- "syscall"
"cos.googlesource.com/cos/tools/src/pkg/utils"
@@ -25,63 +24,20 @@
execCommand = exec.Command
)
-// DisableKernelModuleLocking disables kernel modules signing enforcement and loadpin so that unsigned kernel modules
-// can be loaded to COS kernel.
-func DisableKernelModuleLocking() error {
- log.Info("Checking if third party kernel modules can be installed")
+// CheckKernelModuleSigning checks whether kernel module signing related options present.
+func CheckKernelModuleSigning(kernelCmdline string) bool {
+ log.Info("Checking kernel module signing.")
- mountDir, err := ioutil.TempDir("", "mountdir")
- if err != nil {
- return errors.Wrap(err, "failed to create mount dir")
- }
-
- if err := syscall.Mount(espPartition, mountDir, "vfat", 0, ""); err != nil {
- return errors.Wrap(err, "failed to mount path")
- }
-
- grubCfgPath := filepath.Join(mountDir, "esp/efi/boot/grub.cfg")
- grubCfg, err := ioutil.ReadFile(grubCfgPath)
- if err != nil {
- return errors.Wrapf(err, "failed to read grub config from %s", grubCfgPath)
- }
-
- grubCfgStr := string(grubCfg)
- needReboot := false
for _, kernelOption := range []string{
- "module.sig_enforce",
- "loadpin.enforce",
- "loadpin.enabled",
+ "loadpin.exclude=kernel-module",
+ "modules-load=loadpin_trigger",
+ "module.sig_enforce=1",
} {
- if newGrubCfgStr, needRebootOption := disableKernelOptionFromGrubCfg(kernelOption, grubCfgStr); needRebootOption {
- needReboot = true
- grubCfgStr = newGrubCfgStr
+ if !strings.Contains(kernelCmdline, kernelOption) {
+ return false
}
}
-
- if needReboot {
- log.Info("Modifying grub config to disable module locking.")
- if err := os.Rename(grubCfgPath, grubCfgPath+".orig"); err != nil {
- return errors.Wrapf(err, "failed to rename file %s", grubCfgPath)
- }
- if err := ioutil.WriteFile(grubCfgPath, []byte(grubCfgStr), 0644); err != nil {
- return errors.Wrapf(err, "failed to write to file %s", grubCfgPath)
- }
- } else {
- log.Info("Module locking has been disabled.")
- }
-
- syscall.Sync()
- if err := syscall.Unmount(mountDir, 0); err != nil {
- return err
- }
-
- if needReboot {
- log.Warning("Rebooting")
- if err := syscall.Reboot(syscall.LINUX_REBOOT_CMD_RESTART); err != nil {
- return errors.Wrap(err, "failed to reboot")
- }
- }
- return nil
+ return true
}
// SetCompilationEnv sets compilation environment variables (e.g. CC, CXX) for third-party kernel module compilation.
@@ -119,15 +75,14 @@
}
if empty, _ := utils.IsDirEmpty(destDir); !empty {
log.Info("Found existing toolchain. Skipping download and installation")
- return nil
- }
+ } else {
+ if err := downloader.DownloadToolchain(destDir); err != nil {
+ return errors.Wrap(err, "failed to download toolchain")
+ }
- if err := downloader.DownloadToolchain(destDir); err != nil {
- return errors.Wrap(err, "failed to download toolchain")
- }
-
- if err := exec.Command("tar", "xf", filepath.Join(destDir, toolchainArchive), "-C", destDir).Run(); err != nil {
- return errors.Wrap(err, "failed to extract toolchain archive tarball")
+ if err := exec.Command("tar", "xf", filepath.Join(destDir, toolchainArchive), "-C", destDir).Run(); err != nil {
+ return errors.Wrap(err, "failed to extract toolchain archive tarball")
+ }
}
log.Info("Configuring environment variables for cross-compilation")
diff --git a/src/pkg/cos/cos_test.go b/src/pkg/cos/cos_test.go
index f031491..e28ac2b 100644
--- a/src/pkg/cos/cos_test.go
+++ b/src/pkg/cos/cos_test.go
@@ -207,55 +207,65 @@
func TestDisableKernelOptionFromGrubCfg(t *testing.T) {
for _, tc := range []struct {
- testName string
- kernelOption string
- grubCfg string
- expectedNewGrubCfg string
- expectedNeedReboot bool
+ testName string
+ kernelCmdLine string
+ expectedReturn bool
}{
{
- "LoadPin",
- "loadpin.enabled",
-
- `BOOT_IMAGE=/syslinux/vmlinuz.A init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap ` +
- `loglevel=7 noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 ` +
- `systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 ` +
- `dm_verity.error_behavior=3 dm_verity.max_bios=-1 dm_verity.dev_wait=1 i915.modeset=1 cros_efi root=/dev/dm-0 ` +
- `"dm=1 vroot none ro 1,0 2539520 verity payload=PARTUUID=36547742-9356-EF4E-B9AD-F8DED2F6D087 ` +
- `hashtree=PARTUUID=36547742-9356-EF4E-B9AD-F8DED2F6D087 hashstart=2539520 alg=sha256 ` +
- `root_hexdigest=0ff80250bd97ad47a65e7cd330ab70bcf5013d7a86817dca59fcac77f0ba1a8f ` +
- `salt=414038a6ed9b1f528c327aff4eac16ad5ca4a6699d142ae096e90374af907c34`,
-
- `BOOT_IMAGE=/syslinux/vmlinuz.A init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap ` +
- `loglevel=7 noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 ` +
- `systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 ` +
- `dm_verity.error_behavior=3 dm_verity.max_bios=-1 dm_verity.dev_wait=1 i915.modeset=1 cros_efi loadpin.enabled=0 root=/dev/dm-0 ` +
- `"dm=1 vroot none ro 1,0 2539520 verity payload=PARTUUID=36547742-9356-EF4E-B9AD-F8DED2F6D087 ` +
- `hashtree=PARTUUID=36547742-9356-EF4E-B9AD-F8DED2F6D087 hashstart=2539520 alg=sha256 ` +
- `root_hexdigest=0ff80250bd97ad47a65e7cd330ab70bcf5013d7a86817dca59fcac77f0ba1a8f ` +
- `salt=414038a6ed9b1f528c327aff4eac16ad5ca4a6699d142ae096e90374af907c34`,
- true,
+ testName: "OptionsPresent",
+ kernelCmdLine: `BOOT_IMAGE=/syslinux/vmlinuz.A init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 ` +
+ `noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false ` +
+ `systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 dm_verity.error_behavior=3 dm_verity.max_bios=-1 ` +
+ `dm_verity.dev_wait=1 i915.modeset=1 cros_efi module.sig_enforce=1 modules-load=loadpin_trigger ` +
+ `loadpin.exclude=kernel-module root=/dev/dm-0 "dm=1 vroot none ro 1,0 4077568 verity ` +
+ `payload=PARTUUID=00CE255B-DB42-1E47-A62B-735C7A9A7397 hashtree=PARTUUID=00CE255B-DB42-1E47-A62B-735C7A9A7397 ` +
+ `hashstart=4077568 alg=sha256 root_hexdigest=8a2cfc7097aa7ddfe4101611fad9dd1df59f9c29cfa9b1a5d18f55ae68c9eed5 ` +
+ `salt=65697f247db9275b9e9830d275ca6b830c156187403f6210b2ebcb11c8beaa57"`,
+ expectedReturn: true,
},
{
- "LoadPinEnabled",
- "loadpin.enabled",
- "cros_efi loadpin.enabled=1",
- "cros_efi loadpin.enabled=0",
- true,
+ testName: "MissingLoadpinExclude",
+ kernelCmdLine: `BOOT_IMAGE=/syslinux/vmlinuz.A init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 ` +
+ `noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false ` +
+ `systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 dm_verity.error_behavior=3 dm_verity.max_bios=-1 ` +
+ `dm_verity.dev_wait=1 i915.modeset=1 cros_efi module.sig_enforce=1 modules-load=loadpin_trigger ` +
+ `root=/dev/dm-0 "dm=1 vroot none ro 1,0 4077568 verity ` +
+ `payload=PARTUUID=00CE255B-DB42-1E47-A62B-735C7A9A7397 hashtree=PARTUUID=00CE255B-DB42-1E47-A62B-735C7A9A7397 ` +
+ `hashstart=4077568 alg=sha256 root_hexdigest=8a2cfc7097aa7ddfe4101611fad9dd1df59f9c29cfa9b1a5d18f55ae68c9eed5 ` +
+ `salt=65697f247db9275b9e9830d275ca6b830c156187403f6210b2ebcb11c8beaa57"`,
+ expectedReturn: false,
},
{
- "LoadPinDisabled",
- "loadpin.enabled",
- "cros_efi loadpin.enabled=0",
- "cros_efi loadpin.enabled=0",
- false,
+ testName: "MissingLoadpinTrigger",
+ kernelCmdLine: `BOOT_IMAGE=/syslinux/vmlinuz.A init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 ` +
+ `noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false ` +
+ `systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 dm_verity.error_behavior=3 dm_verity.max_bios=-1 ` +
+ `dm_verity.dev_wait=1 i915.modeset=1 cros_efi module.sig_enforce=1 ` +
+ `loadpin.exclude=kernel-module root=/dev/dm-0 "dm=1 vroot none ro 1,0 4077568 verity ` +
+ `payload=PARTUUID=00CE255B-DB42-1E47-A62B-735C7A9A7397 hashtree=PARTUUID=00CE255B-DB42-1E47-A62B-735C7A9A7397 ` +
+ `hashstart=4077568 alg=sha256 root_hexdigest=8a2cfc7097aa7ddfe4101611fad9dd1df59f9c29cfa9b1a5d18f55ae68c9eed5 ` +
+ `salt=65697f247db9275b9e9830d275ca6b830c156187403f6210b2ebcb11c8beaa57"`,
+ expectedReturn: false,
+ },
+ {
+ testName: "MissingSigEnforce",
+ kernelCmdLine: `BOOT_IMAGE=/syslinux/vmlinuz.A init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 ` +
+ `noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false ` +
+ `systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 dm_verity.error_behavior=3 dm_verity.max_bios=-1 ` +
+ `dm_verity.dev_wait=1 i915.modeset=1 cros_efi modules-load=loadpin_trigger ` +
+ `loadpin.exclude=kernel-module root=/dev/dm-0 "dm=1 vroot none ro 1,0 4077568 verity ` +
+ `payload=PARTUUID=00CE255B-DB42-1E47-A62B-735C7A9A7397 hashtree=PARTUUID=00CE255B-DB42-1E47-A62B-735C7A9A7397 ` +
+ `hashstart=4077568 alg=sha256 root_hexdigest=8a2cfc7097aa7ddfe4101611fad9dd1df59f9c29cfa9b1a5d18f55ae68c9eed5 ` +
+ `salt=65697f247db9275b9e9830d275ca6b830c156187403f6210b2ebcb11c8beaa57"`,
+ expectedReturn: false,
},
} {
- newGrubCfg, needReboot := disableKernelOptionFromGrubCfg(tc.kernelOption, tc.grubCfg)
- if newGrubCfg != tc.expectedNewGrubCfg || needReboot != tc.expectedNeedReboot {
- t.Errorf("%v: Unexpected output:\nexpect grubcfg: %v\ngot grubcfg: %v\nexpect needReboot: %v, got needReboot: %v",
- tc.testName, tc.expectedNewGrubCfg, newGrubCfg, tc.expectedNeedReboot, needReboot)
- }
+ t.Run(tc.testName, func(t *testing.T) {
+ ret := CheckKernelModuleSigning(tc.kernelCmdLine)
+ if ret != tc.expectedReturn {
+ t.Errorf("Unexpected output:%v, expect: %v", ret, tc.expectedReturn)
+ }
+ })
}
}
