Merge "changelog-webapp: Automatic token refresh, secret manager"
diff --git a/go.mod b/go.mod
index b99a99d..a147d73 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,7 @@
go 1.14
require (
+ cloud.google.com/go v0.57.0
cloud.google.com/go/storage v1.10.0
github.com/beevik/etree v1.1.0
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
@@ -19,5 +20,6 @@
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
google.golang.org/api v0.28.0
google.golang.org/grpc v1.29.1
+ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790
google.golang.org/protobuf v1.25.0
)
diff --git a/go.sum b/go.sum
index a6e9e61..c09867d 100644
--- a/go.sum
+++ b/go.sum
@@ -138,6 +138,7 @@
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -163,6 +164,7 @@
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
@@ -325,6 +327,9 @@
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2 h1:FD4wDsP+CQUqh2V12OBOt90pLHVToe58P++fUu3ggV4=
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -408,6 +413,7 @@
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/src/cmd/changelog-webapp/.gcloudignore b/src/cmd/changelog-webapp/.gcloudignore
new file mode 100644
index 0000000..199e6d9
--- /dev/null
+++ b/src/cmd/changelog-webapp/.gcloudignore
@@ -0,0 +1,25 @@
+# This file specifies files that are *not* uploaded to Google Cloud Platform
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+# $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+# Test binary, build with `go test -c`
+*.test
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/README.md b/src/cmd/changelog-webapp/README.md
new file mode 100644
index 0000000..5917052
--- /dev/null
+++ b/src/cmd/changelog-webapp/README.md
@@ -0,0 +1,21 @@
+# COS Changelog Webapp
+A web application that generates a changelog between 2 builds based on the commit difference between them.
+
+## Setup
+### App Engine
+Create a new App Engine project to host this application. Detailed instructions are located [here](https://cloud.google.com/appengine/docs/standard/nodejs/building-app/creating-project).
+
+### Secret Manager
+This application queries secret manager for any information that should not be publicly accessible, such as client secrets and internal instance URLs. Ensure this service is enabled in Google Cloud, and that the App Engine service account has the `Secret Manager Secret Accessor` role in Google Cloud IAM.
+
+## Configuration
+`app.yaml` stores public environment variables used to run the service. For private environment variables, the variable name is stored in Secret Manager instead. This is indicated by the `_NAME` suffix for any environment variable.
+
+For each secret name defined in `app.yaml`, a corresponding secret must be made in Google Secret Manager under the same variable name. See [here](https://cloud.google.com/secret-manager/docs/quickstart#secretmanager-quickstart-web) for more information on managing secrets. Secrets must be made for the Oauth client secret, session secret, internal repository names, and internal Gerrit/Git on Borg URLs.
+
+## Deployment
+Install [Cloud SDK](https://cloud.google.com/sdk/docs) and configure it to use the Google Cloud project you want to deploy to.
+
+Clone the `cos/tools` repository and `cd` to the `src/cmd/changelog-webapp` directory. `app.yaml` should be in this directory.
+
+Run `gcloud app deploy` to deploy the application. You can view the application with `gcloud app browse` after deployment is complete.
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/app.yaml b/src/cmd/changelog-webapp/app.yaml
new file mode 100644
index 0000000..4f005fd
--- /dev/null
+++ b/src/cmd/changelog-webapp/app.yaml
@@ -0,0 +1,23 @@
+runtime: go114
+instance_class: F2
+
+env_variables:
+ # Authentication variables
+ COS_CHANGELOG_PROJECT_ID: "cos-oss-interns-playground" # Used to access project secrets from secret manager
+ OAUTH_CLIENT_ID: "1080882898042-86cke9tvtqd9vrfnd8dj8ub6m0stjv2u.apps.googleusercontent.com"
+ COS_CHANGELOG_CLIENT_SECRET_NAME: "cos-changelog-client-secret"
+ COS_CHANGELOG_SESSION_SECRET_NAME: "cos-changelog-session-secret"
+
+ # Webpage configuration
+ STATIC_BASE_PATH: "src/cmd/changelog-webapp/static/"
+ CHANGELOG_QUERY_SIZE: "50"
+
+ # External sources
+ COS_EXTERNAL_GERRIT_INSTANCE: "cos-review.googlesource.com"
+ COS_EXTERNAL_GOB_INSTANCE: "cos.googlesource.com"
+ COS_EXTERNAL_MANIFEST_REPO: "cos/manifest-snapshots"
+
+ # Internal source names (values are retrieved from secret manager)
+ COS_INTERNAL_GERRIT_INSTANCE_NAME: "cos-internal-gerrit-instance"
+ COS_INTERNAL_GOB_INSTANCE_NAME: "cos-internal-gob-instance"
+ COS_INTERNAL_MANIFEST_REPO_NAME: "cos-internal-manifest-repo"
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/controllers/authHandlers.go b/src/cmd/changelog-webapp/controllers/authHandlers.go
index 56785cf..deb71ac 100644
--- a/src/cmd/changelog-webapp/controllers/authHandlers.go
+++ b/src/cmd/changelog-webapp/controllers/authHandlers.go
@@ -16,22 +16,26 @@
import (
"context"
+ "errors"
"fmt"
"math/rand"
"net/http"
"os"
"time"
+ secretmanager "cloud.google.com/go/secretmanager/apiv1"
"github.com/gorilla/sessions"
log "github.com/sirupsen/logrus"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
+ secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
)
const (
// Session variables
sessionName = "changelog"
sessionKeyLength = 32
+ sessionAge = 84600
// Oauth state generation variables
oauthStateCharset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890"
@@ -40,37 +44,85 @@
var config = &oauth2.Config{
ClientID: os.Getenv("OAUTH_CLIENT_ID"),
- ClientSecret: os.Getenv("OAUTH_CLIENT_SECRET"),
+ ClientSecret: "",
Endpoint: google.Endpoint,
RedirectURL: "https://cos-oss-interns-playground.uc.r.appspot.com/oauth2callback/",
Scopes: []string{"https://www.googleapis.com/auth/gerritcodereview"},
}
+var store *sessions.CookieStore
+var projectID = os.Getenv("COS_CHANGELOG_PROJECT_ID")
+var clientSecretName = os.Getenv("COS_CHANGELOG_CLIENT_SECRET_NAME")
+var sessionSecretName = os.Getenv("COS_CHANGELOG_SESSION_SECRET_NAME")
-var store = sessions.NewCookieStore([]byte(randomString(sessionKeyLength)))
+// ErrorSessionRetrieval indicates that a request has no session, or the
+// session was malformed.
+var ErrorSessionRetrieval = errors.New("No session found")
-func randomString(stringSize int) string {
+func init() {
+ var err error
+ client, err := secretmanager.NewClient(context.Background())
+ if err != nil {
+ log.Fatalf("Failed to setup client: %v", err)
+ }
+ config.ClientSecret, err = getSecret(client, clientSecretName)
+ if err != nil {
+ log.Fatalf("Failed to retrieve secret: %s\n%v", clientSecretName, err)
+ }
+
+ sessionSecret, err := getSecret(client, sessionSecretName)
+ if err != nil {
+ log.Fatalf("Failed to retrieve secret :%s\n%v", sessionSecretName, err)
+ }
+ store = sessions.NewCookieStore([]byte(sessionSecret))
+ store.MaxAge(sessionAge)
+}
+
+// Retrieve secrets stored in Gcloud Secret Manager
+func getSecret(client *secretmanager.Client, secretName string) (string, error) {
+ accessRequest := &secretmanagerpb.AccessSecretVersionRequest{
+ Name: fmt.Sprintf("projects/%s/secrets/%s/versions/latest", projectID, secretName),
+ }
+ result, err := client.AccessSecretVersion(context.Background(), accessRequest)
+ if err != nil {
+ return "", fmt.Errorf("Failed to access secret at %s: %v", accessRequest.Name, err)
+ }
+ return string(result.Payload.Data), nil
+}
+
+func randomString(stringSize int, suffix string) string {
randWithSeed := rand.New(rand.NewSource(time.Now().UnixNano()))
stateArr := make([]byte, stringSize)
for i := range stateArr {
stateArr[i] = oauthStateCharset[randWithSeed.Intn(len(oauthStateCharset))]
}
- return string(stateArr)
+ return string(stateArr) + suffix
}
-// HTTPClient creates an authorized HTTP Client
-func HTTPClient(r *http.Request) (*http.Client, error) {
+func returnURLFromState(state string) string {
+ return state[oauthStateLength:]
+}
+
+// HTTPClient creates an authorized HTTP Client using stored token credentials.
+// Returns error if no session or a malformed session is detected.
+// Otherwise returns an HTTP Client with the stored Oauth access token.
+// If the access token is expired, automatically refresh the token
+func HTTPClient(w http.ResponseWriter, r *http.Request, returnURL string) (*http.Client, error) {
var parsedExpiry time.Time
session, err := store.Get(r, sessionName)
- if err != nil {
- return nil, fmt.Errorf("HTTPClient: No session found with sessionName %s", sessionName)
+ if err != nil || session.IsNew {
+ return nil, ErrorSessionRetrieval
}
for _, key := range []string{"accessToken", "refreshToken", "tokenType", "expiry"} {
if val, ok := session.Values[key]; !ok || val == nil {
- return nil, fmt.Errorf("HTTPClient: Session missing key %s", key)
+ return nil, ErrorSessionRetrieval
}
}
if parsedExpiry, err = time.Parse(time.RFC3339, session.Values["expiry"].(string)); err != nil {
- return nil, fmt.Errorf("HTTPClient: Token expiry is in an incorrect format")
+ return nil, ErrorSessionRetrieval
+ }
+ if parsedExpiry.Before(time.Now()) {
+ log.Debug("HTTPClient: Token expired, calling Oauth flow")
+ HandleLogin(w, r, returnURL)
}
token := &oauth2.Token{
AccessToken: session.Values["accessToken"].(string),
@@ -81,9 +133,12 @@
return config.Client(context.Background(), token), nil
}
-// HandleLogin handles login
-func HandleLogin(w http.ResponseWriter, r *http.Request) {
- state := randomString(oauthStateLength)
+// HandleLogin initiates the Oauth flow.
+func HandleLogin(w http.ResponseWriter, r *http.Request, returnURL string) {
+ state := randomString(oauthStateLength, returnURL)
+ // Ignore store.Get() errors in HandleLogin because an error indicates the
+ // old session could not be deciphered. It returns a new session
+ // regardless.
session, _ := store.Get(r, sessionName)
session.Values["oauthState"] = state
err := session.Save(r, w)
@@ -97,7 +152,9 @@
http.Redirect(w, r, authURL, http.StatusTemporaryRedirect)
}
-// HandleCallback handles callback
+// HandleCallback handles the response from the Oauth callback URL.
+// It verifies response state and populates session with callback values.
+// Redirects to URL stored in the callback state on completion.
func HandleCallback(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
log.Errorf("Could not parse query: %v\n", err)
@@ -113,7 +170,12 @@
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
- if callbackState != session.Values["oauthState"].(string) {
+ if val, ok := session.Values["oauthState"]; !ok || val == nil {
+ http.Redirect(w, r, "/", http.StatusPermanentRedirect)
+ return
+ }
+ sessionState := session.Values["oauthState"].(string)
+ if callbackState != sessionState {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
@@ -136,5 +198,5 @@
return
}
- http.Redirect(w, r, "/", http.StatusPermanentRedirect)
+ http.Redirect(w, r, returnURLFromState(sessionState), http.StatusPermanentRedirect)
}
diff --git a/src/cmd/changelog-webapp/controllers/pageHandlers.go b/src/cmd/changelog-webapp/controllers/pageHandlers.go
index 2304be7..abc425e 100644
--- a/src/cmd/changelog-webapp/controllers/pageHandlers.go
+++ b/src/cmd/changelog-webapp/controllers/pageHandlers.go
@@ -15,6 +15,7 @@
package controllers
import (
+ "context"
"fmt"
"net/http"
"os"
@@ -22,6 +23,7 @@
"strings"
"text/template"
+ secretmanager "cloud.google.com/go/secretmanager/apiv1"
"cos.googlesource.com/cos/tools/src/pkg/changelog"
log "github.com/sirupsen/logrus"
@@ -44,9 +46,20 @@
)
func init() {
- internalInstance = os.Getenv("COS_INTERNAL_INSTANCE")
- internalManifestRepo = os.Getenv("COS_INTERNAL_MANIFEST_REPO")
- externalInstance = os.Getenv("COS_EXTERNAL_INSTANCE")
+ var err error
+ client, err := secretmanager.NewClient(context.Background())
+ if err != nil {
+ log.Fatalf("Failed to setup client: %v", err)
+ }
+ internalInstance, err = getSecret(client, os.Getenv("COS_INTERNAL_GOB_INSTANCE_NAME"))
+ if err != nil {
+ log.Fatalf("Failed to retrieve secret for COS_INTERNAL_GOB_INSTANCE_NAME with key name %s\n%v", os.Getenv("COS_INTERNAL_GOB_INSTANCE_NAME"), err)
+ }
+ internalManifestRepo, err = getSecret(client, os.Getenv("COS_INTERNAL_MANIFEST_REPO_NAME"))
+ if err != nil {
+ log.Fatalf("Failed to retrieve secret for COS_INTERNAL_MANIFEST_REPO_NAME with key name %s\n%v", os.Getenv("COS_INTERNAL_MANIFEST_REPO_NAME"), err)
+ }
+ externalInstance = os.Getenv("COS_EXTERNAL_GOB_INSTANCE")
externalManifestRepo = os.Getenv("COS_EXTERNAL_MANIFEST_REPO")
envQuerySize = getIntVerifiedEnv("CHANGELOG_QUERY_SIZE")
staticBasePath = os.Getenv("STATIC_BASE_PATH")
@@ -195,7 +208,7 @@
changelogTemplate.Execute(w, &changelogPage{QuerySize: envQuerySize})
return
}
- httpClient, err := HTTPClient(r)
+ httpClient, err := HTTPClient(w, r, "/changelog/")
if err != nil {
log.Debug(err)
err = promptLoginTemplate.Execute(w, &promptLoginPage{ActivePage: "changelog"})
diff --git a/src/cmd/changelog-webapp/main.go b/src/cmd/changelog-webapp/main.go
index fb95de8..a0f6fff 100644
--- a/src/cmd/changelog-webapp/main.go
+++ b/src/cmd/changelog-webapp/main.go
@@ -38,7 +38,9 @@
http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(staticBasePath))))
http.HandleFunc("/", controllers.HandleIndex)
- http.HandleFunc("/login/", controllers.HandleLogin)
+ http.HandleFunc("/login/", func(w http.ResponseWriter, r *http.Request) {
+ controllers.HandleLogin(w, r, "/")
+ })
http.HandleFunc("/changelog/", controllers.HandleChangelog)
http.HandleFunc("/oauth2callback/", controllers.HandleCallback)
