Merge "cos-gpu-installer: Add release script"
diff --git a/go.mod b/go.mod
index ad8835c..d2cd902 100644
--- a/go.mod
+++ b/go.mod
@@ -8,8 +8,8 @@
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
github.com/google/go-cmp v0.5.1
github.com/google/subcommands v1.2.0
+ github.com/gorilla/sessions v1.2.0
github.com/julienschmidt/httprouter v1.3.0 // indirect
- github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.6.0
github.com/smartystreets/assertions v1.1.1 // indirect
@@ -17,5 +17,6 @@
github.com/urfave/cli/v2 v2.2.0
go.chromium.org/luci v0.0.0-20200722211809-bab0c30be68b
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+ google.golang.org/api v0.28.0
google.golang.org/protobuf v1.25.0
)
diff --git a/go.sum b/go.sum
index 9fdf0c0..a6e9e61 100644
--- a/go.sum
+++ b/go.sum
@@ -19,6 +19,8 @@
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA=
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
@@ -36,6 +38,8 @@
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
@@ -48,6 +52,7 @@
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -105,32 +110,44 @@
github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE=
github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
+github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
+github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
+github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ=
+github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -144,11 +161,15 @@
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.chromium.org/luci v0.0.0-20200722211809-bab0c30be68b h1:0jYe0tlrrMwhAesVR6p4+3Xryezo1c7jZZYu3+73tuQ=
go.chromium.org/luci v0.0.0-20200722211809-bab0c30be68b/go.mod h1:MIQewVTLvOvc0UioV0JNqTNO/RspKFS0XEeoKrOxsdM=
@@ -303,7 +324,10 @@
golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2 h1:FD4wDsP+CQUqh2V12OBOt90pLHVToe58P++fUu3ggV4=
golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -375,12 +399,16 @@
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/src/cmd/changelog-webapp/controllers/authHandlers.go b/src/cmd/changelog-webapp/controllers/authHandlers.go
new file mode 100644
index 0000000..56785cf
--- /dev/null
+++ b/src/cmd/changelog-webapp/controllers/authHandlers.go
@@ -0,0 +1,140 @@
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controllers
+
+import (
+ "context"
+ "fmt"
+ "math/rand"
+ "net/http"
+ "os"
+ "time"
+
+ "github.com/gorilla/sessions"
+ log "github.com/sirupsen/logrus"
+ "golang.org/x/oauth2"
+ "golang.org/x/oauth2/google"
+)
+
+const (
+ // Session variables
+ sessionName = "changelog"
+ sessionKeyLength = 32
+
+ // Oauth state generation variables
+ oauthStateCharset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890"
+ oauthStateLength = 16
+)
+
+var config = &oauth2.Config{
+ ClientID: os.Getenv("OAUTH_CLIENT_ID"),
+ ClientSecret: os.Getenv("OAUTH_CLIENT_SECRET"),
+ Endpoint: google.Endpoint,
+ RedirectURL: "https://cos-oss-interns-playground.uc.r.appspot.com/oauth2callback/",
+ Scopes: []string{"https://www.googleapis.com/auth/gerritcodereview"},
+}
+
+var store = sessions.NewCookieStore([]byte(randomString(sessionKeyLength)))
+
+func randomString(stringSize int) string {
+ randWithSeed := rand.New(rand.NewSource(time.Now().UnixNano()))
+ stateArr := make([]byte, stringSize)
+ for i := range stateArr {
+ stateArr[i] = oauthStateCharset[randWithSeed.Intn(len(oauthStateCharset))]
+ }
+ return string(stateArr)
+}
+
+// HTTPClient creates an authorized HTTP Client
+func HTTPClient(r *http.Request) (*http.Client, error) {
+ var parsedExpiry time.Time
+ session, err := store.Get(r, sessionName)
+ if err != nil {
+ return nil, fmt.Errorf("HTTPClient: No session found with sessionName %s", sessionName)
+ }
+ for _, key := range []string{"accessToken", "refreshToken", "tokenType", "expiry"} {
+ if val, ok := session.Values[key]; !ok || val == nil {
+ return nil, fmt.Errorf("HTTPClient: Session missing key %s", key)
+ }
+ }
+ if parsedExpiry, err = time.Parse(time.RFC3339, session.Values["expiry"].(string)); err != nil {
+ return nil, fmt.Errorf("HTTPClient: Token expiry is in an incorrect format")
+ }
+ token := &oauth2.Token{
+ AccessToken: session.Values["accessToken"].(string),
+ RefreshToken: session.Values["refreshToken"].(string),
+ TokenType: session.Values["tokenType"].(string),
+ Expiry: parsedExpiry,
+ }
+ return config.Client(context.Background(), token), nil
+}
+
+// HandleLogin handles login
+func HandleLogin(w http.ResponseWriter, r *http.Request) {
+ state := randomString(oauthStateLength)
+ session, _ := store.Get(r, sessionName)
+ session.Values["oauthState"] = state
+ err := session.Save(r, w)
+ if err != nil {
+ log.Errorf("HandleLogin: Error saving key: %v", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ authURL := config.AuthCodeURL(state, oauth2.AccessTypeOffline)
+ http.Redirect(w, r, authURL, http.StatusTemporaryRedirect)
+}
+
+// HandleCallback handles callback
+func HandleCallback(w http.ResponseWriter, r *http.Request) {
+ if err := r.ParseForm(); err != nil {
+ log.Errorf("Could not parse query: %v\n", err)
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+ authCode := r.FormValue("code")
+ callbackState := r.FormValue("state")
+
+ session, err := store.Get(r, sessionName)
+ if err != nil {
+ log.Errorf("HandleCallback: Error retrieving session: %v", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if callbackState != session.Values["oauthState"].(string) {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ token, err := config.Exchange(context.Background(), authCode)
+ if err != nil {
+ log.Errorf("HandleCallback: Error exchanging token: %v", token)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ session.Values["accessToken"] = token.AccessToken
+ session.Values["refreshToken"] = token.RefreshToken
+ session.Values["tokenType"] = token.TokenType
+ session.Values["expiry"] = token.Expiry.Format(time.RFC3339)
+ err = session.Save(r, w)
+ if err != nil {
+ log.Errorf("HandleCallback: Error saving session: %v", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ http.Redirect(w, r, "/", http.StatusPermanentRedirect)
+}
diff --git a/src/cmd/changelog-webapp/controllers/pageHandlers.go b/src/cmd/changelog-webapp/controllers/pageHandlers.go
new file mode 100644
index 0000000..2304be7
--- /dev/null
+++ b/src/cmd/changelog-webapp/controllers/pageHandlers.go
@@ -0,0 +1,241 @@
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controllers
+
+import (
+ "fmt"
+ "net/http"
+ "os"
+ "strconv"
+ "strings"
+ "text/template"
+
+ "cos.googlesource.com/cos/tools/src/pkg/changelog"
+
+ log "github.com/sirupsen/logrus"
+)
+
+const (
+ subjectLen int = 100
+)
+
+var (
+ internalInstance string
+ internalManifestRepo string
+ externalInstance string
+ externalManifestRepo string
+ envQuerySize string
+ staticBasePath string
+ indexTemplate *template.Template
+ changelogTemplate *template.Template
+ promptLoginTemplate *template.Template
+)
+
+func init() {
+ internalInstance = os.Getenv("COS_INTERNAL_INSTANCE")
+ internalManifestRepo = os.Getenv("COS_INTERNAL_MANIFEST_REPO")
+ externalInstance = os.Getenv("COS_EXTERNAL_INSTANCE")
+ externalManifestRepo = os.Getenv("COS_EXTERNAL_MANIFEST_REPO")
+ envQuerySize = getIntVerifiedEnv("CHANGELOG_QUERY_SIZE")
+ staticBasePath = os.Getenv("STATIC_BASE_PATH")
+ indexTemplate = template.Must(template.ParseFiles(staticBasePath + "templates/index.html"))
+ changelogTemplate = template.Must(template.ParseFiles(staticBasePath + "templates/changelog.html"))
+ promptLoginTemplate = template.Must(template.ParseFiles(staticBasePath + "templates/promptLogin.html"))
+}
+
+type changelogData struct {
+ Instance string
+ Source string
+ Target string
+ Additions map[string]*changelog.RepoLog
+ Removals map[string]*changelog.RepoLog
+ Internal bool
+}
+
+type changelogPage struct {
+ Source string
+ Target string
+ QuerySize string
+ RepoTables []*repoTable
+ Internal bool
+}
+
+type repoTable struct {
+ Name string
+ Additions []*repoTableEntry
+ Removals []*repoTableEntry
+ AdditionsLink string
+ RemovalsLink string
+}
+
+type repoTableEntry struct {
+ IsAddition bool
+ SHA *shaAttr
+ Subject string
+ Bugs []*bugAttr
+ AuthorName string
+ CommitterName string
+ CommitTime string
+ ReleaseNote string
+}
+
+type shaAttr struct {
+ Name string
+ URL string
+}
+
+type bugAttr struct {
+ Name string
+ URL string
+}
+
+type promptLoginPage struct {
+ ActivePage string
+}
+
+// getIntVerifiedEnv retrieves an environment variable but checks that it can be
+// converted to int first
+func getIntVerifiedEnv(envName string) string {
+ output := os.Getenv(envName)
+ if _, err := strconv.Atoi(output); err != nil {
+ log.Errorf("getEnvAsInt: Failed to parse env variable %s with value %s: %v",
+ envName, os.Getenv(output), err)
+ }
+ return output
+}
+
+func gobCommitLink(instance, repo, SHA string) string {
+ return fmt.Sprintf("https://%s/%s/+/%s", instance, repo, SHA)
+}
+
+func gobDiffLink(instance, repo, sourceSHA, targetSHA string) string {
+ return fmt.Sprintf("https://%s/%s/+log/%s..%s?n=10000", instance, repo, sourceSHA, targetSHA)
+}
+
+func createRepoTableEntry(instance string, repo string, commit *changelog.Commit, isAddition bool) *repoTableEntry {
+ entry := new(repoTableEntry)
+ entry.IsAddition = isAddition
+ entry.SHA = &shaAttr{Name: commit.SHA[:8], URL: gobCommitLink(instance, repo, commit.SHA)}
+ entry.Subject = commit.Subject
+ if len(entry.Subject) > subjectLen {
+ entry.Subject = entry.Subject[:subjectLen]
+ }
+ entry.Bugs = make([]*bugAttr, len(commit.Bugs))
+ for i, bugURL := range commit.Bugs {
+ name := bugURL[strings.Index(bugURL, "/")+1:]
+ entry.Bugs[i] = &bugAttr{Name: name, URL: "http://" + bugURL}
+ }
+ entry.AuthorName = commit.AuthorName
+ entry.CommitterName = commit.CommitterName
+ entry.CommitTime = commit.CommitTime
+ entry.ReleaseNote = commit.ReleaseNote
+ return entry
+}
+
+func createChangelogPage(data changelogData) *changelogPage {
+ page := &changelogPage{Source: data.Source, Target: data.Target, QuerySize: envQuerySize, Internal: data.Internal}
+ for repoName, repoLog := range data.Additions {
+ table := &repoTable{Name: repoName}
+ for _, commit := range repoLog.Commits {
+ tableEntry := createRepoTableEntry(data.Instance, repoName, commit, true)
+ table.Additions = append(table.Additions, tableEntry)
+ }
+ if _, ok := data.Removals[repoName]; ok {
+ for _, commit := range data.Removals[repoName].Commits {
+ tableEntry := createRepoTableEntry(data.Instance, repoName, commit, false)
+ table.Removals = append(table.Removals, tableEntry)
+ }
+ if data.Removals[repoName].HasMoreCommits {
+ table.RemovalsLink = gobDiffLink(data.Instance, repoName, repoLog.TargetSHA, repoLog.SourceSHA)
+ }
+ }
+ if repoLog.HasMoreCommits {
+ table.AdditionsLink = gobDiffLink(data.Instance, repoName, repoLog.SourceSHA, repoLog.TargetSHA)
+ }
+ page.RepoTables = append(page.RepoTables, table)
+ }
+ // Add remaining repos that had removals but no additions
+ for repoName, repoLog := range data.Removals {
+ if _, ok := data.Additions[repoName]; ok {
+ continue
+ }
+ table := &repoTable{Name: repoName}
+ for _, commit := range repoLog.Commits {
+ tableEntry := createRepoTableEntry(data.Instance, repoName, commit, false)
+ table.Removals = append(table.Removals, tableEntry)
+ }
+ page.RepoTables = append(page.RepoTables, table)
+ if repoLog.HasMoreCommits {
+ table.RemovalsLink = gobDiffLink(data.Instance, repoName, repoLog.TargetSHA, repoLog.SourceSHA)
+ }
+ }
+ return page
+}
+
+// HandleIndex serves the home page
+func HandleIndex(w http.ResponseWriter, r *http.Request) {
+ indexTemplate.Execute(w, nil)
+}
+
+// HandleChangelog serves the changelog page
+func HandleChangelog(w http.ResponseWriter, r *http.Request) {
+ if err := r.ParseForm(); err != nil {
+ changelogTemplate.Execute(w, &changelogPage{QuerySize: envQuerySize})
+ return
+ }
+ httpClient, err := HTTPClient(r)
+ if err != nil {
+ log.Debug(err)
+ err = promptLoginTemplate.Execute(w, &promptLoginPage{ActivePage: "changelog"})
+ if err != nil {
+ log.Errorf("HandleChangelog: error executing promptLogin template: %v", err)
+ }
+ return
+ }
+ source := r.FormValue("source")
+ target := r.FormValue("target")
+ // If no source/target values specified in request, display empty changelog page
+ if source == "" || target == "" {
+ changelogTemplate.Execute(w, &changelogPage{QuerySize: envQuerySize, Internal: true})
+ return
+ }
+ querySize, err := strconv.Atoi(r.FormValue("n"))
+ if err != nil {
+ querySize, _ = strconv.Atoi(envQuerySize)
+ }
+ internal, instance, manifestRepo := false, externalInstance, externalManifestRepo
+ if r.FormValue("internal") == "true" {
+ internal, instance, manifestRepo = true, internalInstance, internalManifestRepo
+ }
+ added, removed, err := changelog.Changelog(httpClient, source, target, instance, manifestRepo, querySize)
+ if err != nil {
+ log.Errorf("HandleChangelog: error retrieving changelog between builds %s and %s on GoB instance: %s with manifest repository: %s\n%v\n",
+ source, target, externalInstance, externalManifestRepo, err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ page := createChangelogPage(changelogData{
+ Instance: instance,
+ Source: source,
+ Target: target,
+ Additions: added,
+ Removals: removed,
+ Internal: internal,
+ })
+ err = changelogTemplate.Execute(w, page)
+ if err != nil {
+ log.Errorf("HandleChangelog: error executing changelog template: %v", err)
+ }
+}
diff --git a/src/cmd/changelog-webapp/main.go b/src/cmd/changelog-webapp/main.go
new file mode 100644
index 0000000..fb95de8
--- /dev/null
+++ b/src/cmd/changelog-webapp/main.go
@@ -0,0 +1,54 @@
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "net/http"
+ "os"
+
+ "cos.googlesource.com/cos/tools/src/cmd/changelog-webapp/controllers"
+
+ log "github.com/sirupsen/logrus"
+)
+
+var (
+ staticBasePath string
+ port string
+)
+
+func init() {
+ staticBasePath = os.Getenv("STATIC_BASE_PATH")
+ port = os.Getenv("PORT")
+}
+
+func main() {
+ log.SetLevel(log.DebugLevel)
+
+ http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(staticBasePath))))
+ http.HandleFunc("/", controllers.HandleIndex)
+ http.HandleFunc("/login/", controllers.HandleLogin)
+ http.HandleFunc("/changelog/", controllers.HandleChangelog)
+ http.HandleFunc("/oauth2callback/", controllers.HandleCallback)
+
+ if port == "" {
+ port = "8081"
+ log.Printf("Defaulting to port %s", port)
+ }
+
+ log.Printf("Listening on port %s", port)
+ if err := http.ListenAndServe(":"+port, nil); err != nil {
+ log.Fatal(err)
+ }
+}
diff --git a/src/cmd/changelog-webapp/static/css/base.css b/src/cmd/changelog-webapp/static/css/base.css
new file mode 100644
index 0000000..1180143
--- /dev/null
+++ b/src/cmd/changelog-webapp/static/css/base.css
@@ -0,0 +1,87 @@
+html,
+body {
+ background-color: #f8f8f8;
+ font-family: 'Roboto', 'Noto', sans-serif;
+ font-weight: 300;
+ margin: 0;
+ overflow-x: hidden;
+ padding: 0;
+}
+
+h1 {
+ font-size: 24;
+}
+
+h2 {
+ font-size: 18;
+}
+
+p,
+p a {
+ font-size: 16;
+}
+
+a:link,
+a:visited {
+ color: blue;
+ text-decoration: none;
+}
+
+label {
+ font-size: 16;
+}
+
+.navbar {
+ align-items: center;
+ background-color: #4285f4;
+ display: flex;
+ height: 50px;
+ padding: 0px 25px;
+ width: 100%;
+}
+
+.navbar-title {
+ color: #f8f8f8;
+ font-size: 24;
+ font-weight: 350;
+}
+
+.sidenav {
+ height: 100%;
+ left: 0;
+ margin-top: 50px;
+ overflow-x: hidden;
+ padding-top: 30px;
+ position: fixed;
+ top: 0;
+ width: 120px;
+ z-index: 1;
+}
+
+.sidenav a {
+ color: #272727;
+ display: block;
+ font-size: 16px;
+ padding: 0px 8px 8px 26px;
+ text-decoration: none;
+ transition: 0.2s;
+}
+
+.sidenav a.active {
+ color: #4285f4;
+ font-weight: bold;
+}
+
+.sidenav a:hover {
+ color: #3d3d3d;
+ font-weight: 550;
+}
+
+.main {
+ margin: 30px 25px 0px 155px;
+}
+
+.text-content {
+ margin: auto;
+ max-width: 700px;
+}
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/static/css/changelog.css b/src/cmd/changelog-webapp/static/css/changelog.css
new file mode 100644
index 0000000..4a7ba7e
--- /dev/null
+++ b/src/cmd/changelog-webapp/static/css/changelog.css
@@ -0,0 +1,110 @@
+th {
+ font-weight: 440;
+ text-align: left;
+ padding-bottom: 0px;
+}
+
+th,
+td {
+ margin-bottom: 5px;
+ padding: 2px;
+}
+
+.changelog-form .text input {
+ height: 24px;
+ margin-bottom: 8px;
+ width: 135px;
+}
+
+.changelog-form .text .submit {
+ height: 24px;
+ width: 59px;
+}
+
+.changelog-form .text label {
+ margin: 4px;
+}
+
+.changelog-form .radio .external {
+ margin-left: 20px;
+}
+
+.sha-legend {
+ margin-top: 10px;
+}
+
+.sha-legend .legend-row {
+ display: flex;
+ flex-flow: row nowrap;
+ font-size: 14px;
+ margin: 3px 0px 3px 4px;
+}
+
+.sha-legend .circle {
+ border-radius: 50%;
+ height: 14px;
+ margin-right: 8px;
+ width: 14px;
+}
+
+.sha-legend .circle.addition {
+ background-color: #adef97;
+}
+
+.sha-legend .circle.removal {
+ background-color: #ffc0c0;
+}
+
+.repo-header {
+ margin: 24px 0px 12px 0px;
+}
+
+.repo-table {
+ border-spacing: 2px;
+ font-size: 14;
+ font-weight: 300;
+ margin: 4px 0px;
+ table-layout: fixed;
+}
+
+.commit-sha {
+ text-align: center;
+ width: 65px;
+}
+
+th.commit-sha {
+ text-align: left;
+}
+
+.addition {
+ background-color: #c2ffae;
+}
+
+.removal {
+ background-color: #ffdada
+}
+
+.commit-subject {
+ width: 520px;
+}
+
+.commit-bugs {
+ width: 115px;
+}
+
+.commit-author {
+ width: 150px;
+}
+
+.commit-committer {
+ width: 150px;
+}
+
+.commit-time {
+ width: 120px;
+}
+
+.gob-link {
+ font-size: 14;
+ padding-left: 2px;
+}
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/static/templates/changelog.html b/src/cmd/changelog-webapp/static/templates/changelog.html
new file mode 100644
index 0000000..93a3ca0
--- /dev/null
+++ b/src/cmd/changelog-webapp/static/templates/changelog.html
@@ -0,0 +1,122 @@
+<html>
+<head>
+ <meta name="description" content="Google COS build changelog">
+ <link rel="stylesheet" href="/static/css/base.css">
+ <link rel="stylesheet" href="/static/css/changelog.css">
+</head>
+<body>
+ <div class="navbar">
+ <p class="navbar-title">Container Optimized OS</p>
+ </div>
+ <div class="sidenav">
+ <a href="/">Home</a>
+ <a class="active" href="/changelog/">Changelog</a>
+ <a href="/locatecl/">Locate CL</a>
+ <a href="/login/">Login</a>
+ </div>
+ <div class="main">
+ <h1>Search Changelog</h1>
+ <form class="changelog-form" action="/changelog">
+ <div class="text">
+ <label for="source">From </label>
+ {{if (ne .Source "")}}
+ <input type="text" class="source" name="source" placeholder="COS build number" value={{.Source}} autocomplete="off">
+ {{else}}
+ <input type="text" class="source" name="source" placeholder="COS build number" autocomplete="off">
+ {{end}}
+ <label for="target"> to </label>
+ {{if (ne .Target "")}}
+ <input type="text" class="target" name="target" placeholder="COS build number" value={{.Target}} autocomplete="off" required>
+ {{else}}
+ <input type="text" class="target" name="target" placeholder="COS build number" autocomplete="off" required>
+ {{end}}
+ <input type="hidden" name="n" value={{.QuerySize}}>
+ <input class="submit" type="submit" value="Submit"><br>
+ </div>
+ <div class="radio">
+ {{if .Internal}}
+ <input type="radio" class="internal" name="internal" value="true" checked>
+ <label for="internal"> Internal </label>
+ <input type="radio" class="external" name="internal" value="false">
+ <label for="external"> External </label>
+ {{else}}
+ <input type="radio" class="internal" name="internal" value="true">
+ <label for="internal"> Internal </label>
+ <input type="radio" class="external" name="internal" value="false" checked>
+ <label for="external"> External </label>
+ {{end}}
+ </div>
+ </form>
+ {{if (and (ne .Target "") (ne .Source ""))}}
+ <div class="sha-legend">
+ <div class="legend-row">
+ <div class="circle addition"></div>
+ <span>Commits introduced to build {{.Target}} since build {{.Source}}</span><br>
+ </div>
+ <div class="legend-row">
+ <div class="circle removal"></div>
+ <span>Commits introduced to build {{.Source}} since build {{.Target}}</span>
+ </div>
+ </div>
+ {{end}}
+ {{range $table := .RepoTables}}
+ <h2 class="repo-header"> {{$table.Name}} </h2>
+ <table class="repo-table">
+ <tr>
+ <th class="commit-sha">SHA</th>
+ <th class="commit-subject">Subject</th>
+ <th class="commit-bugs">Bugs</th>
+ <th class="commit-author">Author</th>
+ <th class="commit-committer">Committer</th>
+ <th class="commit-time">Date</th>
+ <th class="commit-release-notes">Release Notes</th>
+ </tr>
+ </table>
+ <table class="repo-table">
+ {{range $commit := $table.Additions}}
+ <tr>
+ <td class="commit-sha addition">
+ <a href={{$commit.SHA.URL}} target="_blank">{{$commit.SHA.Name}}</a>
+ </td>
+ <td class="commit-subject">{{$commit.Subject}}</td>
+ <td class="commit-bugs">
+ {{range $bugAttr := $commit.Bugs}}
+ <a href={{$bugAttr.URL}} target="_blank">{{$bugAttr.Name}}</a>
+ {{end}}
+ </td>
+ <td class="commit-author">{{$commit.AuthorName}}</td>
+ <td class="commit-committer">{{$commit.CommitterName}}</td>
+ <td class="commit-time">{{$commit.CommitTime}}</td>
+ <td class="commit-release-notes">{{$commit.ReleaseNote}}</td>
+ </tr>
+ {{end}}
+ </table>
+ {{if (ne $table.AdditionsLink "")}}
+ <a class="gob-link" href={{$table.AdditionsLink}} target="_blank">Show more commits</a>
+ {{end}}
+ <table class="repo-table">
+ {{range $commit := $table.Removals}}
+ <tr>
+ <td class="commit-sha removal">
+ <a href={{$commit.SHA.URL}} target="_blank">{{$commit.SHA.Name}}</a>
+ </td>
+ <td class="commit-subject">{{$commit.Subject}}</td>
+ <td class="commit-bugs">
+ {{range $bugAttr := $commit.Bugs}}
+ <a href={{$bugAttr.URL}} target="_blank">{{$bugAttr.Name}}</a>
+ {{end}}
+ </td>
+ <td class="commit-author">{{$commit.AuthorName}}</td>
+ <td class="commit-committer">{{$commit.CommitterName}}</td>
+ <td class="commit-time">{{$commit.CommitTime}}</td>
+ <td class="commit-release-notes">{{$commit.ReleaseNote}}</td>
+ </tr>
+ {{end}}
+ </table>
+ {{if (ne $table.RemovalsLink "")}}
+ <a class="gob-link" href={{$table.RemovalsLink}} target="_blank">Show more commits</a>
+ {{end}}
+ {{end}}
+ </div>
+</body>
+</html>
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/static/templates/index.html b/src/cmd/changelog-webapp/static/templates/index.html
new file mode 100644
index 0000000..69e5e69
--- /dev/null
+++ b/src/cmd/changelog-webapp/static/templates/index.html
@@ -0,0 +1,30 @@
+<html>
+<head>
+ <meta name="description" content="Google COS build information">
+ <link rel="stylesheet" href="/static/css/base.css">
+</head>
+<body>
+ <div class="navbar">
+ <p class="navbar-title">Container Optimized OS</p>
+ </div>
+ <div class="sidenav">
+ <a class="active" href="/">Home</a>
+ <a href="/changelog/">Changelog</a>
+ <a href="/locatecl/">Locate CL</a>
+ <a href="/login/">Login</a>
+ </div>
+ <div class="main">
+ <div class="text-content">
+ <h1>Container-Optimized OS</h1>
+ <h2>A small, secure, stand alone VM image for building on top of Google Cloud</h2>
+ <p>Container-Optimized OS is an operating system image for your Compute Engine VMs that is optimized for
+ running
+ Docker containers. With Container-Optimized OS, you can bring up your Docker containers on Google Cloud
+ Platform
+ quickly, efficiently, and securely. Container-Optimized OS is maintained by Google and is based on the
+ open
+ source Chromium OS project.</p>
+ </div>
+ </div>
+</body>
+</html>
\ No newline at end of file
diff --git a/src/cmd/changelog-webapp/static/templates/promptLogin.html b/src/cmd/changelog-webapp/static/templates/promptLogin.html
new file mode 100644
index 0000000..6091473
--- /dev/null
+++ b/src/cmd/changelog-webapp/static/templates/promptLogin.html
@@ -0,0 +1,33 @@
+<html>
+<head>
+ <meta name="description" content="Google COS build information">
+ <link rel="stylesheet" href="/static/css/base.css">
+</head>
+<body>
+ <div class="navbar">
+ <p class="navbar-title">Container Optimized OS</p>
+ </div>
+ <div class="sidenav">
+ <a href="/">Home</a>
+ {{if (eq .ActivePage "changelog")}}
+ <a class="active" href="/changelog/">Changelog</a>
+ {{else}}
+ <a href="/changelog/">Changelog</a>
+ {{end}}
+ {{if (eq .ActivePage "locateCl")}}
+ <a class="active" href="/locatecl/">Locate CL</a>
+ {{else}}
+ <a href="/locatecl/">Locate CL</a>
+ {{end}}
+ <a href="/login/">Login</a>
+ </div>
+ <div class="main">
+ <div class="text-content">
+ <h1>Please sign in to use this feature</h1>
+ <p>This application requires OAuth authentication to communicate with Google Git repositories. Your account data
+ will not be used for any other purposes. To continue, please sign in by clicking <a href="/login/">here</a>.
+ </p>
+ </div>
+ </div>
+</body>
+</html>
\ No newline at end of file
diff --git a/src/cmd/changelog/main.go b/src/cmd/changelog/main.go
index 3eb0731..4531d36 100755
--- a/src/cmd/changelog/main.go
+++ b/src/cmd/changelog/main.go
@@ -57,7 +57,7 @@
return oauth2.NewClient(oauth2.NoContext, creds.TokenSource), nil
}
-func writeChangelogAsJSON(source string, target string, changes map[string][]*changelog.Commit) error {
+func writeChangelogAsJSON(source string, target string, changes map[string]*changelog.RepoLog) error {
fileName := fmt.Sprintf("%s -> %s.json", source, target)
log.Infof("Writing changelog to %s\n", fileName)
jsonData, err := json.MarshalIndent(changes, "", " ")
@@ -76,7 +76,7 @@
if err != nil {
return fmt.Errorf("generateChangelog: failed to create http client: \n%v", err)
}
- sourceToTargetChanges, targetToSourceChanges, err := changelog.Changelog(httpClient, source, target, instance, manifestRepo)
+ sourceToTargetChanges, targetToSourceChanges, err := changelog.Changelog(httpClient, source, target, instance, manifestRepo, -1)
if err != nil {
return fmt.Errorf("generateChangelog: error retrieving changelog between builds %s and %s on GoB instance: %s with manifest repository: %s\n%v",
source, target, instance, manifestRepo, err)
diff --git a/src/cmd/changelog/main_test.py b/src/cmd/changelog/main_test.py
index 251a9ad..9751fd3 100644
--- a/src/cmd/changelog/main_test.py
+++ b/src/cmd/changelog/main_test.py
@@ -62,8 +62,8 @@
data = json.load(f)
if len(data) == 0:
return False
- for repo, commit_log in data.items():
- for commit in commit_log:
+ for repoName, repoLog in data.items():
+ for commit in repoLog['Commits']:
if not check_commit_schema(commit):
return False
return True
@@ -71,10 +71,15 @@
class TestCLIApplication(unittest.TestCase):
- def test_build(self):
+ def setUp(self):
process = subprocess.run(["go", "build", "-o", "changelog","main.go"])
assert process.returncode == 0
+ def tearDown(self):
+ delete_logs("15000.0.0", "15055.0.0")
+ delete_logs("15050.0.0", "15056.0.0")
+ delete_logs("15056.0.0", "15056.0.0")
+
def test_basic_run(self):
source = "15050.0.0"
target = "15056.0.0"
diff --git a/src/cmd/cos_image_analyzer/README.md b/src/cmd/cos_image_analyzer/README.md
new file mode 100644
index 0000000..602df84
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/README.md
@@ -0,0 +1,92 @@
+# COS Image Analyzer
+
+COS Image Analyzer is a Linux based command line tool written in Go that can analyze a single COS image or compare two COS images for its relevant binary and package information. The differences include the filesystem difference, OS configuration difference, package version upgrades, etc. The tool has customizable flags below for different use cases. Output is default to the terminal.
+
+## Usage
+Run ./cos_image_analyzer -h/-help
+```
+NAME
+ cos_image_analyzer - finds all meaningful differences of two COS Images (binary and package differences).
+ If only one image is passed in, its binary info and package info will be returned.
+
+SYNOPSIS
+ %s [-local] FILE-1 [FILE-2] (default true)
+ FILE - the local file path to the DOS/MBR boot sector file of your image (Ex: disk.raw)
+ Ex: %s image-cos-77-12371-273-0/disk.raw image-cos-81-12871-119-0/disk.raw
+
+ %s -local -binary=Sysctl-settings,OS-config -package=false image-cos-77-12371-273-0/disk.raw
+
+ %s -gcs GCS-PATH-1 [GCS-PATH-2]
+ GCS-PATH - the GCS "gs://bucket/object" path for the COS Image ("object" is type .tar.gz)
+ Ex: %s -gcs gs://my-bucket/cos-images/cos-77-12371-273-0.tar.gz gs://my-bucket/cos-images/cos-81-12871-119-0.tar.gz
+
+
+DESCRIPTION
+ Input Flags:
+ -local (default true, flag is optional)
+ input is one or two DOS/MBR disk file on the local filesystem. If the images are downloaded from
+ Google Cloud as a tarball, decompress the tarball first then pass the disk.raw file to the program.
+ -gcs
+ input is one or two objects stored on Google Cloud Storage of type (.tar.gz). This flag temporarily downloads,
+ unzips, and loop device mounts the images into this tool's directory.
+ To download images from Google Cloud Storage, you need to pass a service account credential to the program.
+ Folllow https://cloud.google.com/docs/authentication/production#create_service_account to create a service account and
+ download the service account key. Then point environment variable GOOGLE_APPLICATION_CREDENTIALS to the key file then
+ run the program.
+
+ Difference Flags:
+ -binary (string)
+ specify which type of binary difference to show. Types "Version", "BuildID", "Kernel-command-line",
+ "Partition-structure", "Sysctl-settings", and "Kernel-configs" are supported for one and two image. "Rootfs",
+ "Stateful-partition", and "OS-config" are only supported for two images. To list multiple types separate by
+ comma. To NOT list any binary difference, set flag to "false". (default all types)
+ -package
+ specify whether to show package difference. Shows addition/removal of packages and package version updates.
+ To NOT list any package difference, set flag to false. (default false)
+
+ Attribute Flags
+ -verbose
+ include flag to increase verbosity of Rootfs, Stateful-partition, and OS-config differences. See -compress-rootfs and
+ -compress-stateful flags descriptions for the directories that are compressed by default.
+ -compress-rootfs (string)
+ to customize which directories are compressed in a non-verbose Rootfs and OS-config difference output, provide a local
+ file path to a .txt file. Format of the file must be one root file path per line with an ending back slash and no commas.
+ By default the directory(s) that are compressed during a diff are /bin/, /lib/modules/, /lib64/, /usr/libexec/, /usr/bin/,
+ /usr/sbin/, /usr/lib64/, /usr/share/zoneinfo/, /usr/share/git/, /usr/lib/, /sbin/, /etc/ssh/, /etc/os-release/ and
+ /etc/package_list/.
+ -compress-stateful (string)
+ to customize which directories are compressed in a non-verbose Stateful-partition difference output, provide a local
+ file path to a .txt file. Format of file must be one root file path per line with no commas. By default the directory(s)
+ that are compressed during a diff are /var_overlay/db/.
+
+ Output Flags:
+ -output (string)
+ Specify format of output. Only "terminal" stdout or "json" object is supported. (default "terminal")
+
+OUTPUT
+ Based on the "-output" flag. Either "terminal" stdout or machine readable "json" format.
+
+NOTE
+ The root permission is needed for this program because it needs to mount images into your local filesystem to calculate difference.
+```
+
+## Code Layout
+
+main.go - The controller of execution: Parse images, find binary and package difference, output to the user, and then clean up.
+
+internal/input/ - Package dedicated to parsing input flags and arguments to setup for execution. Temporary directory is create and all necessary partitions are mounted.
+
+internal/binary/ - Collects, determines, and formats all binary differences.
+
+internal/packagediff/ - Collects, determines, and formats all package differences.
+
+internal/output/ - Final formatting of output at the end of execution.
+
+internal/utilities/ - Helper functions used throughout the project (GCS_download, logical helpers, etc).
+
+internal/testdata/ - Testing data for all packages.
+
+
+## Documentation
+
+go/cos-image-analyzer
diff --git a/src/cmd/cos_image_analyzer/internal/binary/binarydiff.go b/src/cmd/cos_image_analyzer/internal/binary/binarydiff.go
deleted file mode 100644
index c4bfa5e..0000000
--- a/src/cmd/cos_image_analyzer/internal/binary/binarydiff.go
+++ /dev/null
@@ -1,49 +0,0 @@
-package binary
-
-import (
- "fmt"
-
- "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/utilities"
-)
-
-// Global variables
-var (
- // Command-line path strings
- // /etc/os-release is the file describing COS versioning
- etcOSRelease = "/etc/os-release"
-)
-
-// BinaryDiff is a tool that finds all binary differneces of two COS images
-// (COS version, rootfs, kernel command line, stateful parition, ...)
-//
-// Input: (string) img1Path - The path to the root directory for COS image1
-// (string) img2Path - The path to the root directory for COS image2
-//
-// Output: (stdout) terminal ouput - All differences printed to the terminal
-func BinaryDiff(img1Path, img2Path string) error {
- fmt.Println("================== Binary Differences ==================")
-
- // COS Verison Difference
- fmt.Println("--------- COS Verison Difference ---------")
- verMap1, err := utilities.ReadFileToMap(img1Path+etcOSRelease, "=")
- if err != nil {
- return err
- }
- verMap2, err := utilities.ReadFileToMap(img2Path+etcOSRelease, "=")
- if err != nil {
- return err
- }
-
- // Compare Version (Major)
- _, err = utilities.CmpMapValues(verMap1, verMap2, "VERSION")
- if err != nil {
- return err
- }
- // Compare BUILD_ID (Minor)
- _, err = utilities.CmpMapValues(verMap1, verMap2, "BUILD_ID")
- if err != nil {
- return err
- }
-
- return nil
-}
diff --git a/src/cmd/cos_image_analyzer/internal/binary/diff.go b/src/cmd/cos_image_analyzer/internal/binary/diff.go
new file mode 100644
index 0000000..73f6ce4
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/binary/diff.go
@@ -0,0 +1,358 @@
+package binary
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/input"
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/utilities"
+)
+
+// Global variables
+var (
+ // Command-line path strings
+ // /etc is the OS configurations directory
+ etc = "/etc/"
+
+ // /etc/os-release is the file describing COS versioning
+ etcOSRelease = "/etc/os-release"
+)
+
+// Differences is a intermediate Struct used to store all binary differences
+// Field names are pre-defined in parse_input.go and will be cross-checked with -binary flag.
+type Differences struct {
+ Version []string
+ BuildID []string
+ Rootfs string
+ OSConfigs map[string]string
+ Stateful string
+ PartitionStructure string
+ KernelConfigs string
+ KernelCommandLine map[string]string
+ SysctlSettings string
+}
+
+// versionDiff calculates the Version difference of two images
+func (d *Differences) versionDiff(image1, image2 *input.ImageInfo) {
+ if image1.Version != image2.Version {
+ d.Version = []string{image1.Version, image2.Version}
+ }
+}
+
+// buildDiff calculates the BuildID difference of two images
+func (d *Differences) buildDiff(image1, image2 *input.ImageInfo) {
+ if image1.BuildID != image2.BuildID {
+ d.BuildID = []string{image1.BuildID, image2.BuildID}
+ }
+}
+
+// rootfsDiff calculates the Root FS difference of two images
+func (d *Differences) rootfsDiff(image1, image2 *input.ImageInfo, flagInfo *input.FlagInfo) error {
+ rootfsDiff, err := directoryDiff(image1.RootfsPartition3, image2.RootfsPartition3, "rootfs", flagInfo.Verbose, flagInfo.CompressRootfsSlice)
+ if err != nil {
+ return fmt.Errorf("fail to diff Rootfs partitions %v and %v: %v", image1.RootfsPartition3, image2.RootfsPartition3, err)
+ }
+ d.Rootfs = rootfsDiff
+ return nil
+}
+
+// osConfigDiff calculates the OsConfig difference of two images
+func (d *Differences) osConfigDiff(image1, image2 *input.ImageInfo, flagInfo *input.FlagInfo) error {
+ mapOfEtcEntries, err := findOSConfigs(image1, image2) // Get map of /etc entries for both images
+ if err != nil {
+ return fmt.Errorf("failed to find OS Configs: %v", err)
+ }
+ output := make(map[string]string)
+ for etcEntryName, img := range mapOfEtcEntries {
+ etcEntryPath := filepath.Join(etc, etcEntryName) + "/"
+ if flagInfo.Verbose || !utilities.InArray(etcEntryPath, flagInfo.CompressRootfsSlice) { // Only diff if Verbose or etcEntry is not in CompressRootfs.txt
+ currentImage := img
+ if img != "" { // Unique /etc entry in Image 1 or Image2
+ output[etcEntryPath] += "Only in " + img + "/rootfs/etc: " + etcEntryName
+ } else { // Shared /etc entry in Image 1 and Image 2
+ osConfigDiff, err := pureDiff(filepath.Join(image1.RootfsPartition3, etcEntryPath), filepath.Join(image2.RootfsPartition3, etcEntryPath))
+ if err != nil {
+ return fmt.Errorf("fail to take \"diff -r --no-dereference\" on %v: %v", etcEntryPath, err)
+ }
+ currentImage = image1.TempDir
+ output[etcEntryPath] = osConfigDiff
+ }
+
+ fullPath := filepath.Join(currentImage, "/rootfs/", etcEntryPath)
+ entryFile, err := os.Stat(fullPath)
+ if err != nil {
+ return fmt.Errorf("failed to get info on file %v: %v", fullPath, err)
+ }
+ if output[etcEntryPath] != "" {
+ if entryFile.IsDir() {
+ output[etcEntryPath] = "Configs for directory " + etcEntryPath + "\n" + output[etcEntryPath]
+ } else {
+ output[etcEntryPath] = "Configs for file " + etcEntryPath + "\n" + output[etcEntryPath]
+ }
+ }
+ }
+ }
+ d.OSConfigs = output
+ return nil
+}
+
+// statefulDiff calculates the stateful partition difference of two images
+func (d *Differences) statefulDiff(image1, image2 *input.ImageInfo, flagInfo *input.FlagInfo) error {
+ statefulDiff, err := directoryDiff(image1.StatePartition1, image2.StatePartition1, "stateful", flagInfo.Verbose, flagInfo.CompressStatefulSlice)
+ if err != nil {
+ return fmt.Errorf("failed to diff stateful partitions %v and %v: %v", image1.StatePartition1, image2.StatePartition1, err)
+ }
+ d.Stateful = statefulDiff
+ return nil
+}
+
+// partitionStructureDiff calculates the Version difference of two images
+func (d *Differences) partitionStructureDiff(image1, image2 *input.ImageInfo) error {
+ if image2.TempDir != "" {
+ partitionStructureDiff, err := pureDiff(image1.PartitionFile, image2.PartitionFile)
+ if err != nil {
+ return fmt.Errorf("fail to compare both image's \"partitions.txt\" file: %v", err)
+ }
+ d.PartitionStructure = partitionStructureDiff
+ } else {
+ image1Structure, err := ioutil.ReadFile(image1.PartitionFile)
+ if err != nil {
+ return fmt.Errorf("failed to read partition file of image %v: %v", image1.TempDir, err)
+ }
+ d.PartitionStructure = string(image1Structure)
+ }
+ return nil
+}
+
+// kernelConfigsDiff calculates the kernel configs difference of two images
+func (d *Differences) kernelConfigsDiff(image1, image2 *input.ImageInfo) error {
+ if image2.TempDir != "" {
+ kernelConfigsDiff, err := pureDiff(image1.KernelConfigsFile, image2.KernelConfigsFile)
+ if err != nil {
+ return fmt.Errorf("fail to compare the two image's kernel configs files: %v", err)
+ }
+ d.KernelConfigs = kernelConfigsDiff
+ } else {
+ image1KernelConfigs, err := ioutil.ReadFile(image1.KernelConfigsFile)
+ if err != nil {
+ return fmt.Errorf("failed to read kernel configs file of image %v: %v", image1.TempDir, err)
+ }
+ d.KernelConfigs = string(image1KernelConfigs)
+ }
+ return nil
+}
+
+// kernelCommandLineDiff calculates the kernel commad line difference of two images
+func (d *Differences) kernelCommandLineDiff(image1, image2 *input.ImageInfo) error {
+ output := make(map[string]string)
+ if image2.TempDir != "" {
+ mapImage1 := getKclMap(strings.Fields(image1.KernelCommandLine))
+ mapImage2 := getKclMap(strings.Fields(image2.KernelCommandLine))
+
+ for key1, value1 := range mapImage1 {
+ if value2, ok := mapImage2[key1]; !ok { // Unique KCL parameter in image1
+ if value1 != "" {
+ output[key1] = "d\n" + "< " + key1 + "=" + value1
+ } else {
+ output[key1] = "d\n" + "< " + key1
+ }
+ } else if value2 != value1 { // Image1 and Image2 KCL parameter values differ
+ output[key1] = "c\n" + "< " + key1 + "=" + value1 + "\n---\n> " + key1 + "=" + value2
+ }
+ }
+ for key2, value2 := range mapImage2 {
+ if _, ok := mapImage1[key2]; !ok { // Unique KCL parameter in image2
+ if value2 != "" {
+ output[key2] = "a\n" + "> " + key2 + "=" + value2
+ } else {
+ output[key2] = "a\n" + "> " + key2
+ }
+ }
+ }
+ } else {
+ output["Image1 KCL"] = image1.KernelCommandLine
+ }
+ d.KernelCommandLine = output
+ return nil
+}
+
+// sysctlSettingsDiff calculates the sysctl Settings difference of two images
+func (d *Differences) sysctlSettingsDiff(image1, image2 *input.ImageInfo) error {
+ if image2.TempDir != "" {
+ sysctlSettingsDiff, err := pureDiff(image1.SysctlSettingsFile, image2.SysctlSettingsFile)
+ if err != nil {
+ return fmt.Errorf("fail to compare the two image's sysctl settings files: %v", err)
+ }
+ d.SysctlSettings = sysctlSettingsDiff
+ } else {
+ image1SysctlSettings, err := ioutil.ReadFile(image1.SysctlSettingsFile)
+ if err != nil {
+ return fmt.Errorf("failed to convert image 1's %v file to string: %v", image1.SysctlSettingsFile, err)
+ }
+ d.SysctlSettings = string(image1SysctlSettings)
+ }
+ return nil
+}
+
+// FormatVersionDiff returns a formated string of the version difference
+func (d *Differences) FormatVersionDiff() string {
+ if len(d.Version) == 2 {
+ if d.Version[1] != "" {
+ return "----------Version----------\n< " + d.Version[0] + "\n> " + d.Version[1] + "\n\n"
+ }
+ return "----------Version----------\n" + d.Version[0] + "\n\n"
+ }
+ return ""
+}
+
+// FormatBuildIDDiff returns a formated string of the build difference
+func (d *Differences) FormatBuildIDDiff() string {
+ if len(d.BuildID) == 2 {
+ if d.BuildID[1] != "" {
+ return "----------BuildID----------\n< " + d.BuildID[0] + "\n> " + d.BuildID[1] + "\n\n"
+ }
+ return "----------BuildID----------\n" + d.BuildID[0] + "\n\n"
+ }
+ return ""
+}
+
+// FormatRootfsDiff returns a formated string of the rootfs difference
+func (d *Differences) FormatRootfsDiff() string {
+ if d.Rootfs != "" {
+ return "----------RootFS----------\n" + d.Rootfs + "\n\n"
+ }
+ return ""
+}
+
+// FormatStatefulDiff returns a formated string of the stateful partition difference
+func (d *Differences) FormatStatefulDiff() string {
+ if d.Stateful != "" {
+ return "----------Stateful Partition----------\n" + d.Stateful + "\n\n"
+ }
+ return ""
+}
+
+// FormatOSConfigDiff returns a formated string of the OS Config difference
+func (d *Differences) FormatOSConfigDiff() string {
+ if len(d.OSConfigs) > 0 {
+ osConfigDifference := "----------OS Configurations----------\n"
+ keys := make([]string, 0)
+ for k := range d.OSConfigs {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for _, k := range keys {
+ if d.OSConfigs[k] != "" {
+ osConfigDifference += d.OSConfigs[k] + "\n\n"
+ }
+ }
+ return osConfigDifference
+ }
+ return ""
+}
+
+// FormatPartitionStructureDiff returns a formated string of the partition structure difference
+func (d *Differences) FormatPartitionStructureDiff() string {
+ if d.PartitionStructure != "" {
+ return "----------Partition Structure----------\n" + d.PartitionStructure + "\n\n"
+ }
+ return ""
+}
+
+// FormatKernelConfigsDiff returns a formated string of the kernel configs difference
+func (d *Differences) FormatKernelConfigsDiff() string {
+ if d.KernelConfigs != "" {
+ return "----------Kernel Configs----------\n" + d.KernelConfigs + "\n\n"
+ }
+ return ""
+}
+
+// FormatKernelCommandLineDiff returns a formated string of the KCL difference
+func (d *Differences) FormatKernelCommandLineDiff() string {
+ if len(d.KernelCommandLine) > 0 {
+ kclDifference := "----------Kernel Command Line----------\n"
+ keys := make([]string, 0)
+ for k := range d.KernelCommandLine {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for _, k := range keys {
+ if d.KernelCommandLine[k] != "" {
+ kclDifference += d.KernelCommandLine[k] + "\n\n"
+ }
+ }
+ return kclDifference
+ }
+ return ""
+}
+
+// FormatSysctlSettingsDiff returns a formated string of the Sysctrl settings difference
+func (d *Differences) FormatSysctlSettingsDiff() string {
+ if d.SysctlSettings != "" {
+ return "----------Sysctl settings----------\n" + d.SysctlSettings + "\n\n"
+ }
+ return ""
+}
+
+// Diff is a tool that finds all binary differences of two COS images
+// (COS version, rootfs, kernel command line, stateful partition, ...)
+// Input:
+// (*ImageInfo) image1 - A struct that will store binary info for image1
+// (*ImageInfo) image2 - A struct that will store binary info for image2
+// (*FlagInfo) flagInfo - A struct that holds input preference from the user
+// Output:
+// (*Differences) BinaryDiff - A struct that will store the binary differences
+func Diff(image1, image2 *input.ImageInfo, flagInfo *input.FlagInfo) (*Differences, error) {
+ BinaryDiff := &Differences{}
+
+ if utilities.InArray("Version", flagInfo.BinaryTypesSelected) {
+ BinaryDiff.versionDiff(image1, image2)
+ }
+ if utilities.InArray("BuildID", flagInfo.BinaryTypesSelected) {
+ BinaryDiff.buildDiff(image1, image2)
+ }
+
+ if utilities.InArray("Partition-structure", flagInfo.BinaryTypesSelected) {
+ if err := BinaryDiff.partitionStructureDiff(image1, image2); err != nil {
+ return BinaryDiff, fmt.Errorf("Failed to get Partition-structure difference: %v", err)
+ }
+ }
+ if utilities.InArray("Kernel-configs", flagInfo.BinaryTypesSelected) {
+ if err := BinaryDiff.kernelConfigsDiff(image1, image2); err != nil {
+ return BinaryDiff, fmt.Errorf("failed to get Kernel-configs difference: %v", err)
+ }
+ }
+ if utilities.InArray("Kernel-command-line", flagInfo.BinaryTypesSelected) {
+ if err := BinaryDiff.kernelCommandLineDiff(image1, image2); err != nil {
+ return BinaryDiff, fmt.Errorf("failed to get Kernel-command-line difference: %v", err)
+ }
+ }
+ if utilities.InArray("Sysctl-settings", flagInfo.BinaryTypesSelected) {
+ if err := BinaryDiff.sysctlSettingsDiff(image1, image2); err != nil {
+ return BinaryDiff, fmt.Errorf("failed to get Sysctl-settings difference: %v", err)
+ }
+ }
+
+ if image2.TempDir != "" {
+ if utilities.InArray("Rootfs", flagInfo.BinaryTypesSelected) {
+ if err := BinaryDiff.rootfsDiff(image1, image2, flagInfo); err != nil {
+ return BinaryDiff, fmt.Errorf("Failed to get Roofs difference: %v", err)
+ }
+ }
+ if utilities.InArray("OS-config", flagInfo.BinaryTypesSelected) {
+ if err := BinaryDiff.osConfigDiff(image1, image2, flagInfo); err != nil {
+ return BinaryDiff, fmt.Errorf("Failed to get OS-config difference: %v", err)
+ }
+ }
+ if utilities.InArray("Stateful-partition", flagInfo.BinaryTypesSelected) {
+ if err := BinaryDiff.statefulDiff(image1, image2, flagInfo); err != nil {
+ return BinaryDiff, fmt.Errorf("Failed to get Stateful-partition difference: %v", err)
+ }
+ }
+ }
+ return BinaryDiff, nil
+}
diff --git a/src/cmd/cos_image_analyzer/internal/binary/diff_test.go b/src/cmd/cos_image_analyzer/internal/binary/diff_test.go
new file mode 100644
index 0000000..ab7a995
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/binary/diff_test.go
@@ -0,0 +1,335 @@
+package binary
+
+import (
+ "testing"
+
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/input"
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/utilities"
+)
+
+// test Diff function
+func TestDiff(t *testing.T) {
+ testCompressRootfsSlice := []string{"/proc/", "/usr/lib/", "/util/", "/etc/os-release/", "/etc/sysctl.d/"}
+ testCompressStatefulSlice := []string{"/var_overlay/"}
+
+ // Rootfs test data
+ testVerboseRootfsDiff := `Files ../testdata/image1/rootfs/lib64/python.txt and ../testdata/image2/rootfs/lib64/python.txt differ
+Files ../testdata/image1/rootfs/proc/security/access.conf and ../testdata/image2/rootfs/proc/security/access.conf differ
+Files ../testdata/image1/rootfs/proc/security/configs and ../testdata/image2/rootfs/proc/security/configs differ
+Only in ../testdata/image1/rootfs/usr/lib: usr-lib-image1
+Only in ../testdata/image2/rootfs/usr/lib: usr-lib-image2`
+ testBriefRootfsDiff := `Files ../testdata/image1/rootfs/lib64/python.txt and ../testdata/image2/rootfs/lib64/python.txt differ
+Files in ../testdata/image1/rootfs/proc and ../testdata/image2/rootfs/proc differ
+Unique files in ../testdata/image1/rootfs/usr/lib
+Unique files in ../testdata/image2/rootfs/usr/lib`
+
+ // OS Config test data
+ testVerboseOSConfig := map[string]string{
+ "/etc/docker/": `Configs for directory /etc/docker/
+diff -r --no-dereference ../testdata/image1/rootfs/etc/docker/credentials.txt ../testdata/image2/rootfs/etc/docker/credentials.txt
+1,2c1,2
+< Name: docker.10.2.4
+< job: makes micro kernels
+\ No newline at end of file
+---
+> Name: docker.10.2.1
+> job: makes macro kernels
+\ No newline at end of file
+Only in ../testdata/image1/rootfs/etc/docker/util: docker.txt
+Only in ../testdata/image1/rootfs/etc/docker/util: lib32
+Only in ../testdata/image2/rootfs/etc/docker/util: lib64`,
+ "/etc/os-release/": `Configs for file /etc/os-release/
+1c1
+< BUILD_ID=12871.119.0
+---
+> BUILD_ID=12371.273.0
+3c3
+< KERNEL_COMMIT_ID=fa84f12c6d738af9486e69a006a57df923f9476a
+---
+> KERNEL_COMMIT_ID=5d4ffd91281840f7a118143d77fbefb02e87943c
+5c5
+< VERSION_ID=81
+---
+> VERSION_ID=77
+8c8
+< VERSION=81
+---
+> VERSION=77`,
+ "/etc/sysctl.d/": `Configs for directory /etc/sysctl.d/
+diff -r --no-dereference ../testdata/image1/rootfs/etc/sysctl.d/00-sysctl.conf ../testdata/image2/rootfs/etc/sysctl.d/00-sysctl.conf
+8c8
+< net.ipv4.conf.all.rp_filter = 1
+---
+> net.ipv4.conf.all.rp_filter = 2
+11c11,14
+< net.ipv4.tcp_slow_start_after_idle = 0
+\ No newline at end of file
+---
+> net.ipv4.tcp_slow_start_after_idle = 1
+>
+> # dumby variable
+> net.ipv4.conf = 2
+\ No newline at end of file`}
+
+ testBriefOSConfig := map[string]string{
+ "/etc/docker/": `Configs for directory /etc/docker/
+diff -r --no-dereference ../testdata/image1/rootfs/etc/docker/credentials.txt ../testdata/image2/rootfs/etc/docker/credentials.txt
+1,2c1,2
+< Name: docker.10.2.4
+< job: makes micro kernels
+\ No newline at end of file
+---
+> Name: docker.10.2.1
+> job: makes macro kernels
+\ No newline at end of file
+Only in ../testdata/image1/rootfs/etc/docker/util: docker.txt
+Only in ../testdata/image1/rootfs/etc/docker/util: lib32
+Only in ../testdata/image2/rootfs/etc/docker/util: lib64`}
+
+ // Stateful test data
+ testVerboseStatefulDiff := `Only in ../testdata/image1/stateful/dev_image: image1_dev.txt
+Only in ../testdata/image2/stateful/dev_image: image2_dev.txt
+Only in ../testdata/image1/stateful/var_overlay/db: image1_data.txt
+Only in ../testdata/image2/stateful/var_overlay/db: image2_data.txt`
+ testBriefStatefulDiff := `Only in ../testdata/image1/stateful/dev_image: image1_dev.txt
+Only in ../testdata/image2/stateful/dev_image: image2_dev.txt
+Unique files in ../testdata/image1/stateful/var_overlay
+Unique files in ../testdata/image2/stateful/var_overlay`
+
+ // Partition Structure data
+ testPartitionStructure := `1c1
+< Disk /img_disks/cos_81_12871_119_disk/disk.raw: 20971520 sectors, 10.0 GiB
+---
+> Disk /img_disks/cos_77_12371_273_disk/disk.raw: 20971520 sectors, 10.0 GiB
+3c3
+< Disk identifier (GUID): 0274E604-5DE3-5E4E-A4FD-F4D00FBBD7AA
+---
+> Disk identifier (GUID): AB9719F2-3174-4F46-8079-1CF470D2D9BC
+11c11
+< 1 8704000 18874476 4.8 GiB 8300 STATE
+---
+> 1 8704000 18874476 4.8 GiB 0700 STATE
+18c18
+< 8 86016 118783 16.0 MiB 8300 OEM
+---
+> 8 86016 118783 16.0 MiB 0700 OEM`
+
+ // Kernel configs data
+ testKernelConfigsImage1 := `#
+# Compiler: Chromium OS 10.0_pre377782_p20200113-r10 clang version 10.0.0 (/var/cache/chromeos-cache/distfiles/host/egit-src/llvm-project 4e8231b5cf0f5f62c7a51a857e29f5be5cb55734)
+#
+CONFIG_GCC_VERSION=0
+CONFIG_CC_IS_CLANG=y
+CONFIG_CLANG_VERSION=100000
+
+#
+# General setup
+#
+CONFIG_INIT_ENV_ARG_LIMIT=32
+CONFIG_LOCALVERSION=""
+CONFIG_BUILD_SALT=""
+CONFIG_HAVE_KERNEL_GZIP=y`
+ testKernelConfigsDiff := `2c2
+< # Compiler: Chromium OS 10.0_pre377782_p20200113-r10 clang version 10.0.0 (/var/cache/chromeos-cache/distfiles/host/egit-src/llvm-project 4e8231b5cf0f5f62c7a51a857e29f5be5cb55734)
+---
+> # Compiler: Chromium OS 9.0_pre361749_p20190714-r4 clang version 9.0.0 (/var/cache/chromeos-cache/distfiles/host/egit-src/llvm-project c11de5eada2decd0a495ea02676b6f4838cd54fb) (based on LLVM 9.0.0svn)
+6c6
+< CONFIG_CLANG_VERSION=100000
+---
+> CONFIG_CLANG_VERSION=90000
+11a12
+> # CONFIG_COMPILE_TEST is not set
+12a14
+> # CONFIG_LOCALVERSION_AUTO is not set
+14c16
+< CONFIG_HAVE_KERNEL_GZIP=y
+\ No newline at end of file
+---
+> CONFIG_HAVE_KERNEL_GZIP=y`
+
+ // Kernel Command Line data
+ kclImage1 := `linux /syslinux/vmlinuz.A init=/usr/lib/systemd/systemd boot=local rootwait ro dm_verity.dev_wait=50`
+ kclImage2 := `linux /syslinux/vmlinuz.A init=/usr/lib32/systemd/ ro dm_verity.dev_wait=1 i915.modeset=1 cros_efi`
+ testKCLImage1 := map[string]string{"Image1 KCL": kclImage1}
+ testKCLDiff := map[string]string{
+ "boot": `d
+< boot=local`,
+ "cros_efi": `a
+> cros_efi`,
+ "dm_verity.dev_wait": `c
+< dm_verity.dev_wait=50
+---
+> dm_verity.dev_wait=1`,
+ "i915.modeset": `a
+> i915.modeset=1`,
+ "init": `c
+< init=/usr/lib/systemd/systemd
+---
+> init=/usr/lib32/systemd/`,
+ "rootwait": `d
+< rootwait`}
+
+ // Sysctl settings
+ testSysctlSettingsImage1 := `# /etc/sysctl.conf
+# Look in /proc/sys/ for all the things you can setup.
+#
+
+# Enables source route verification
+net.ipv4.conf.default.rp_filter = 1
+# Enable reverse path
+net.ipv4.conf.all.rp_filter = 1
+
+# Disable shrinking the cwnd when connection is idle
+net.ipv4.tcp_slow_start_after_idle = 0`
+ testSysctlSettingsDiff := `8c8
+< net.ipv4.conf.all.rp_filter = 1
+---
+> net.ipv4.conf.all.rp_filter = 2
+11c11,14
+< net.ipv4.tcp_slow_start_after_idle = 0
+\ No newline at end of file
+---
+> net.ipv4.tcp_slow_start_after_idle = 1
+>
+> # dumby variable
+> net.ipv4.conf = 2
+\ No newline at end of file`
+ for _, tc := range []struct {
+ Image1 *input.ImageInfo
+ Image2 *input.ImageInfo
+ FlagInfo *input.FlagInfo
+ want *Differences
+ }{ // Version and BuildID difference tests
+ {Image1: &input.ImageInfo{RootfsPartition3: "../testdata/image1/rootfs/", Version: "81", BuildID: "12871.119.0"},
+ Image2: &input.ImageInfo{},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Version", "BuildID"}},
+ want: &Differences{Version: []string{"81", ""}, BuildID: []string{"12871.119.0", ""}}},
+
+ {Image1: &input.ImageInfo{RootfsPartition3: "../testdata/image1/rootfs/", Version: "81", BuildID: "12871.119.0"},
+ Image2: &input.ImageInfo{RootfsPartition3: "../testdata/image2/rootfs/", Version: "77", BuildID: "12371.273.0"},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Version", "BuildID"}},
+ want: &Differences{Version: []string{"81", "77"}, BuildID: []string{"12871.119.0", "12371.273.0"}}},
+
+ {Image1: &input.ImageInfo{RootfsPartition3: "../testdata/image1/rootfs/", Version: "81", BuildID: "12871.119.0"},
+ Image2: &input.ImageInfo{RootfsPartition3: "../testdata/image1/rootfs/", Version: "81", BuildID: "12871.119.1"},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Version", "BuildID"}},
+ want: &Differences{Version: []string{}, BuildID: []string{"12871.119.0", "12871.119.1"}}},
+
+ {Image1: &input.ImageInfo{RootfsPartition3: "../testdata/image1/rootfs/", Version: "81", BuildID: "12871.119.0"},
+ Image2: &input.ImageInfo{RootfsPartition3: "../testdata/image1/rootfs/", Version: "81", BuildID: "12871.119.0"},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Version", "BuildID"}},
+ want: &Differences{}},
+
+ // Rootfs difference test
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", RootfsPartition3: "../testdata/image1/rootfs/"},
+ Image2: &input.ImageInfo{},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Rootfs"}},
+ want: &Differences{}},
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", RootfsPartition3: "../testdata/image1/rootfs/"},
+ Image2: &input.ImageInfo{TempDir: "../testdata/image2", RootfsPartition3: "../testdata/image2/rootfs/"},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Rootfs"}, Verbose: true, CompressRootfsSlice: testCompressRootfsSlice},
+ want: &Differences{Rootfs: testVerboseRootfsDiff}},
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", RootfsPartition3: "../testdata/image1/rootfs/"},
+ Image2: &input.ImageInfo{TempDir: "../testdata/image2", RootfsPartition3: "../testdata/image2/rootfs/"},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Rootfs"}, Verbose: false, CompressRootfsSlice: testCompressRootfsSlice},
+ want: &Differences{Rootfs: testBriefRootfsDiff}},
+
+ // OS Config difference test
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", RootfsPartition3: "../testdata/image1/rootfs/"},
+ Image2: &input.ImageInfo{},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"OS-config"}},
+ want: &Differences{}},
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", RootfsPartition3: "../testdata/image1/rootfs/"},
+ Image2: &input.ImageInfo{TempDir: "../testdata/image2", RootfsPartition3: "../testdata/image2/rootfs/"},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"OS-config"}, Verbose: true, CompressRootfsSlice: testCompressRootfsSlice},
+ want: &Differences{OSConfigs: testVerboseOSConfig}},
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", RootfsPartition3: "../testdata/image1/rootfs/"},
+ Image2: &input.ImageInfo{TempDir: "../testdata/image2", RootfsPartition3: "../testdata/image2/rootfs/"},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"OS-config"}, Verbose: false, CompressRootfsSlice: testCompressRootfsSlice},
+ want: &Differences{OSConfigs: testBriefOSConfig}},
+
+ // Stateful difference test
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", StatePartition1: "../testdata/image1/stateful/"},
+ Image2: &input.ImageInfo{},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Stateful-partition"}},
+ want: &Differences{}},
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", StatePartition1: "../testdata/image1/stateful/"},
+ Image2: &input.ImageInfo{TempDir: "../testdata/image2", StatePartition1: "../testdata/image2/stateful/"},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Stateful-partition"}, Verbose: true, CompressStatefulSlice: testCompressStatefulSlice},
+ want: &Differences{Stateful: testVerboseStatefulDiff}},
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", StatePartition1: "../testdata/image1/stateful/"},
+ Image2: &input.ImageInfo{TempDir: "../testdata/image2", StatePartition1: "../testdata/image2/stateful/"},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Stateful-partition"}, Verbose: false, CompressStatefulSlice: testCompressStatefulSlice},
+ want: &Differences{Stateful: testBriefStatefulDiff}},
+
+ // Partition Structure
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", PartitionFile: "../testdata/image1/partitions.txt"},
+ Image2: &input.ImageInfo{TempDir: "../testdata/image2", PartitionFile: "../testdata/image2/partitions.txt"},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Partition-structure"}},
+ want: &Differences{PartitionStructure: testPartitionStructure}},
+
+ // Kernel Configs
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", KernelConfigsFile: "../testdata/image1/usr/src/linux-headers-4.19.112+/.config"},
+ Image2: &input.ImageInfo{},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Kernel-configs"}},
+ want: &Differences{KernelConfigs: testKernelConfigsImage1}},
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", KernelConfigsFile: "../testdata/image1/usr/src/linux-headers-4.19.112+/.config"},
+ Image2: &input.ImageInfo{TempDir: "../testdata/image2", KernelConfigsFile: "../testdata/image2/usr/src/linux-headers-4.19.112+/.config"},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Kernel-configs"}},
+ want: &Differences{KernelConfigs: testKernelConfigsDiff}},
+
+ // Kernel command line
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", KernelCommandLine: kclImage1},
+ Image2: &input.ImageInfo{},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Kernel-command-line"}},
+ want: &Differences{KernelCommandLine: testKCLImage1}},
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", KernelCommandLine: kclImage1},
+ Image2: &input.ImageInfo{TempDir: "../testdata/image2", KernelCommandLine: kclImage2},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Kernel-command-line"}},
+ want: &Differences{KernelCommandLine: testKCLDiff}},
+
+ // Sysctl settings
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", SysctlSettingsFile: "../testdata/image1/rootfs/etc/sysctl.d/00-sysctl.conf"},
+ Image2: &input.ImageInfo{},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Sysctl-settings"}},
+ want: &Differences{SysctlSettings: testSysctlSettingsImage1}},
+ {Image1: &input.ImageInfo{TempDir: "../testdata/image1", SysctlSettingsFile: "../testdata/image1/rootfs/etc/sysctl.d/00-sysctl.conf"},
+ Image2: &input.ImageInfo{TempDir: "../testdata/image2", SysctlSettingsFile: "../testdata/image2/rootfs/etc/sysctl.d/00-sysctl.conf"},
+ FlagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Sysctl-settings"}},
+ want: &Differences{SysctlSettings: testSysctlSettingsDiff}},
+ } {
+ got, _ := Diff(tc.Image1, tc.Image2, tc.FlagInfo)
+
+ if !utilities.EqualArrays(tc.want.Version, got.Version) {
+ t.Fatalf("Diff expected version %v, got: %v", tc.want.Version, got.Version)
+ }
+ if !utilities.EqualArrays(tc.want.BuildID, got.BuildID) {
+ t.Fatalf("Diff expected BuildID %v, got: %v", tc.want.BuildID, got.BuildID)
+ }
+ if tc.want.Rootfs != got.Rootfs {
+ t.Fatalf("Diff expected Rootfs diff \n%v\ngot:\n%v", tc.want.Rootfs, got.Rootfs)
+ }
+ for etcEntry := range tc.want.OSConfigs {
+ if res, _ := utilities.CmpMapValues(tc.want.OSConfigs, got.OSConfigs, etcEntry); res != 0 {
+ t.Fatalf("Diff expected OSConfigs \n%v\ngot:\n%v", tc.want.OSConfigs, got.OSConfigs)
+ }
+ }
+ if tc.want.Stateful != got.Stateful {
+ t.Fatalf("Diff expected stateful diff \n%v\ngot:\n%v", tc.want.Stateful, got.Stateful)
+ }
+ if tc.want.PartitionStructure != got.PartitionStructure {
+ t.Fatalf("Diff expected partition structure diff \n$%v$\ngot:\n$%v$", tc.want.PartitionStructure, got.PartitionStructure)
+ }
+ for kclParameter, diff := range tc.want.KernelCommandLine {
+ if diff != got.KernelCommandLine[kclParameter] {
+ t.Fatalf("Diff expected kernel command line \n$%v$\ngot:\n$%v$", tc.want.KernelCommandLine, got.KernelCommandLine)
+ }
+ }
+ if tc.want.KernelConfigs != got.KernelConfigs {
+ t.Fatalf("Diff expected kernel configs diff \n$%v$\ngot:\n$%v$", tc.want.KernelConfigs, got.KernelConfigs)
+ }
+ if tc.want.SysctlSettings != got.SysctlSettings {
+ t.Fatalf("Diff expected sysctl settings \n$%v$\ngot:\n$%v$", tc.want.SysctlSettings, got.SysctlSettings)
+ }
+ }
+}
diff --git a/src/cmd/cos_image_analyzer/internal/binary/helpers.go b/src/cmd/cos_image_analyzer/internal/binary/helpers.go
new file mode 100644
index 0000000..136c4d9
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/binary/helpers.go
@@ -0,0 +1,221 @@
+package binary
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strings"
+
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/input"
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/utilities"
+)
+
+// findOSConfigs creates a map of all /etc entries in both images
+// Format: {etcEntry: ""} if etcEntry is shared in both images
+// {etcEntry: imageName} if etcEntry is unique to "imageName"
+func findOSConfigs(image1, image2 *input.ImageInfo) (map[string]string, error) {
+ etcFiles1, err := ioutil.ReadDir(image1.RootfsPartition3 + etc)
+ if err != nil {
+ return map[string]string{}, fmt.Errorf("fail to read contents of directory %v: %v", image1.RootfsPartition3+etc, err)
+ }
+ etcEntries1 := []string{}
+ for _, f := range etcFiles1 {
+ if _, err := os.Readlink(filepath.Join(image1.RootfsPartition3, etc, f.Name())); err != nil {
+ etcEntries1 = append(etcEntries1, f.Name())
+ }
+ }
+
+ etcFiles2, err := ioutil.ReadDir(image2.RootfsPartition3 + etc)
+ if err != nil {
+ return map[string]string{}, fmt.Errorf("fail to read contents of directory %v: %v", image2.RootfsPartition3+etc, err)
+ }
+ etcEntries2 := []string{}
+ for _, f := range etcFiles2 {
+ if _, err := os.Readlink(filepath.Join(image1.RootfsPartition3, etc, f.Name())); err != nil {
+ etcEntries2 = append(etcEntries2, f.Name())
+ }
+ }
+
+ osConfigsMap := make(map[string]string)
+ for _, elem1 := range etcEntries1 {
+ if !utilities.InArray(elem1, etcEntries2) { // Unique file or directory in image 1
+ osConfigsMap[elem1] = image1.TempDir
+ } else { // Common /etc files or directories for image 1 and 2
+ osConfigsMap[elem1] = ""
+ }
+ }
+ for _, elem2 := range etcEntries2 {
+ if _, ok := osConfigsMap[elem2]; !ok { // Unique file or directory in image 2
+ osConfigsMap[elem2] = image2.TempDir
+ }
+ }
+ return osConfigsMap, nil
+}
+
+// getKclMap converts a kernel commad line tokenized slice into a map where the keys are
+// kernel Command line parameters and the values are the parameter's value (if it exists)
+// Format: {kclParameter: value} if kclparameter follows form "param=value"
+// {kclParameter: ""}if kclparameter follows form "param"
+func getKclMap(input []string) map[string]string {
+ output := make(map[string]string)
+ for _, elem := range input {
+ if strings.Contains(elem, "=") { // KCl parameter follows form "parameter=value"
+ if startOfEquals := strings.Index(elem, "="); startOfEquals >= 0 {
+ key, value := elem[:startOfEquals], ""
+ if startOfEquals != len(elem)-1 {
+ value = elem[startOfEquals+1:]
+ }
+ output[key] = value
+ }
+ } else { // KCl parameter follows form "parameter"
+ output[elem] = ""
+ }
+ }
+ return output
+}
+
+// findDiffDir finds the directory name from the "diff" command
+// for the "Only in [file path]" case.
+// Input:
+// (string) line - A single line of output from the "diff -rq" command
+// (string) dir1 - Path to directory 1
+// (string) dir2 - Path to directory 2
+// Output:
+// (string) dir1 or dir2 - The directory found in "line"
+// (bool) ok - Flag to indicate a directory has been found
+func findDiffDir(line, dir1, dir2 string) (string, bool) {
+ lineSplit := strings.Split(line, " ")
+ if len(lineSplit) < 3 {
+ return "", false
+ }
+
+ for _, word := range lineSplit {
+ if strings.Contains(word, dir1) && strings.Contains(word, dir2) {
+ return "", false
+ }
+ if strings.Contains(word, dir1) {
+ return dir1, true
+ }
+ if strings.Contains(word, dir2) {
+ return dir2, true
+ }
+ }
+ return "", false
+}
+
+// compressString compresses lines of a string that fit a pattern
+// Input:
+// (string) dir1 - Path to directory 1
+// (string) dir2 - Path to directory 2
+// (string) root - Name of the root for directories 1 and 2
+// (string) input - The string to be filtered
+// ([]string) patterns - The list of patterns to be filtered out
+// Output:
+// (string) output - The compacted version of the input string
+func compressString(dir1, dir2, root, input string, patterns []string) (string, error) {
+ patternMap := utilities.SliceToMapStr(patterns)
+
+ lines := strings.Split(string(input), "\n")
+ for i, line := range lines {
+ for pat, count := range patternMap {
+ fullPattern := filepath.Join(root, pat)
+ fileInPattern := fullPattern + "/"
+ onlyInPattern := fullPattern + ":"
+ if strings.Contains(line, fileInPattern) || strings.Contains(line, onlyInPattern) {
+ lineSplit := strings.Split(line, " ")
+ if len(lineSplit) < 3 {
+ continue
+ }
+
+ typeOfDiff := lineSplit[0]
+ if typeOfDiff == "Files" || typeOfDiff == "Symbolic" || typeOfDiff == "File" {
+ if strings.Contains(count, "differentFilesFound") {
+ lines[i] = ""
+ continue
+ }
+ lines[i] = "Files in " + filepath.Join(dir1, pat) + " and " + filepath.Join(dir2, pat) + " differ"
+ patternMap[pat] += "differentFilesFound"
+ } else if typeOfDiff == "Only" {
+ if strings.Contains(count, "dir1_UniqueFileFound") && strings.Contains(count, "dir2_UniqueFileFound") {
+ lines[i] = ""
+ continue
+ }
+ if onlyDir, ok := findDiffDir(line, dir1, dir2); ok {
+ if onlyDir == dir1 {
+ if !strings.Contains(count, "dir1_UniqueFileFound") {
+ lines[i] = "Unique files in " + filepath.Join(onlyDir, pat)
+ patternMap[pat] += "dir1_UniqueFileFound"
+ } else {
+ lines[i] = ""
+ continue
+ }
+ } else if onlyDir == dir2 {
+ if !strings.Contains(count, "dir2_UniqueFileFound") {
+ lines[i] = "Unique files in " + filepath.Join(onlyDir, pat)
+ patternMap[pat] += "dir2_UniqueFileFound"
+ } else {
+ lines[i] = ""
+ continue
+ }
+ }
+ }
+ } else { // Compress any other diff output not described above
+ lines[i] = ""
+ }
+ }
+ }
+ }
+
+ output := strings.Join(lines, "\n")
+ output = regexp.MustCompile(`[\t\r\n]+`).ReplaceAllString(strings.TrimSpace(output), "\n")
+ return output, nil
+}
+
+// DirectoryDiff finds the recursive file difference between two directories.
+// If verbose is true return full difference, else compress based on compressedDirs
+// Input:
+// (string) dir1 - Path to directory 1
+// (string) dir2 - Path to directory 2
+// (string) root - Name of the root for directories 1 and 2
+// ([]string) compressedDirs - List of directories to compress by
+// (bool) verbose - Flag that determines whether to show full or compressed difference
+// Output:
+// (string) diff - The file difference output of the "diff" command
+func directoryDiff(dir1, dir2, root string, verbose bool, compressedDirs []string) (string, error) {
+ var cmd *exec.Cmd
+ if root == "rootfs" { // Only exclude "/etc" for Rootfs difference
+ cmd = exec.Command("sudo", "diff", "--no-dereference", "-rq", "-x", "etc", dir1, dir2)
+ } else {
+ cmd = exec.Command("sudo", "diff", "--no-dereference", "-rq", dir1, dir2)
+ }
+ diff, err := cmd.Output()
+ if exitError, ok := err.(*exec.ExitError); ok {
+ if exitError.ExitCode() == 2 {
+ return "", fmt.Errorf("failed to call 'diff' command on directories %v and %v: %v", dir1, dir2, err)
+ }
+ }
+
+ diffStr := strings.TrimSuffix(string(diff), "\n")
+ if verbose {
+ return diffStr, nil
+ }
+ compressedDiffStr, err := compressString(dir1, dir2, root, diffStr, compressedDirs)
+ if err != nil {
+ return "", fmt.Errorf("failed to call compress 'diff' output between %v and %v: %v", dir1, dir2, err)
+ }
+ return compressedDiffStr, nil
+}
+
+// pureDiff returns the output of a normal diff between two files or directories
+func pureDiff(input1, input2 string) (string, error) {
+ diff, err := exec.Command("sudo", "diff", "-r", "--no-dereference", input1, input2).Output()
+ if exitError, ok := err.(*exec.ExitError); ok {
+ if exitError.ExitCode() == 2 {
+ return "", fmt.Errorf("failed to call 'diff' on %v and %v: %v", input1, input2, err)
+ }
+ }
+ return strings.TrimSuffix(string(diff), "\n"), nil
+}
diff --git a/src/cmd/cos_image_analyzer/internal/binary/helpers_test.go b/src/cmd/cos_image_analyzer/internal/binary/helpers_test.go
new file mode 100644
index 0000000..da5bbea
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/binary/helpers_test.go
@@ -0,0 +1,86 @@
+package binary
+
+import (
+ "testing"
+)
+
+// test DirectoryDiff function
+func TestDirectoryDiff(t *testing.T) {
+ testVerboseOutput := `Files ../testdata/image1/rootfs/lib64/python.txt and ../testdata/image2/rootfs/lib64/python.txt differ
+Files ../testdata/image1/rootfs/proc/security/access.conf and ../testdata/image2/rootfs/proc/security/access.conf differ
+Files ../testdata/image1/rootfs/proc/security/configs and ../testdata/image2/rootfs/proc/security/configs differ
+Only in ../testdata/image1/rootfs/usr/lib: usr-lib-image1
+Only in ../testdata/image2/rootfs/usr/lib: usr-lib-image2`
+ testBriefOutput := `Files ../testdata/image1/rootfs/lib64/python.txt and ../testdata/image2/rootfs/lib64/python.txt differ
+Files in ../testdata/image1/rootfs/proc and ../testdata/image2/rootfs/proc differ
+Unique files in ../testdata/image1/rootfs/usr/lib
+Unique files in ../testdata/image2/rootfs/usr/lib`
+
+ for _, tc := range []struct {
+ dir1 string
+ dir2 string
+ root string
+ verbose bool
+ compressedDirs []string
+ want string
+ }{
+ {dir1: "../testdata/image1/rootfs/", dir2: "../testdata/image2/rootfs/", root: "rootfs", verbose: true, compressedDirs: []string{"/proc/", "/usr/lib/"}, want: testVerboseOutput},
+ {dir1: "../testdata/image1/rootfs/", dir2: "../testdata/image2/rootfs/", root: "rootfs", verbose: false, compressedDirs: []string{"/proc/", "/usr/lib/"}, want: testBriefOutput},
+ } {
+ got, _ := directoryDiff(tc.dir1, tc.dir2, tc.root, tc.verbose, tc.compressedDirs)
+ if got != tc.want {
+ t.Fatalf("directoryDiff expected:\n%v\ngot:\n%v", tc.want, got)
+ }
+ }
+}
+
+// test PureDiff function
+func TestPureDiff(t *testing.T) {
+ testOutput1 := `1c1
+< testing 123 can you hear me?
+---
+> testing 456 can you hear me?`
+ testOutput2 := `1c1
+< These are not the configs you are looking for
+---
+> These are the configs you are looking for`
+ for _, tc := range []struct {
+ input1 string
+ input2 string
+ want string
+ }{
+ {input1: "../testdata/image1/rootfs/proc/security/access.conf", input2: "../testdata/image2/rootfs/proc/security/access.conf", want: testOutput1},
+ {input1: "../testdata/image1/rootfs/proc/security/configs", input2: "../testdata/image2/rootfs/proc/security/configs", want: testOutput2},
+ {input1: "../testdata/image1/rootfs/proc/security/lib-image1", input2: "../testdata/image2/rootfs/proc/security/lib-image2", want: ""},
+ } {
+ got, _ := pureDiff(tc.input1, tc.input2)
+ if got != tc.want {
+ t.Fatalf("PureDiff expected:\n%v\ngot:\n%v", tc.want, got)
+ }
+ }
+}
+
+// test getKclMap function
+func TestGetKclMap(t *testing.T) {
+
+ for _, tc := range []struct {
+ input []string
+ want map[string]string
+ }{
+ {input: []string{"linux", "/syslinux/vmlinuz.A", "init=/usr/lib/systemd/systemd", "boot=local", "rootwait", "ro", "dm_verity.dev_wait=50"},
+ want: map[string]string{"syslinux/vmlinuz.A": "", "boot": "local", "dm_verity.dev_wait": "50", "init": "/usr/lib/systemd/systemd", "linux": "", "ro": "", "rootwait": ""}},
+
+ {input: []string{"linux", "/syslinux/vmlinuz.A", "init=/usr/lib32/systemd/", "ro", "dm_verity.dev_wait=2", "i915.modeset=1", "cros_efi"},
+ want: map[string]string{"syslinux/vmlinuz.A": "", "dm_verity.dev_wait": "2", "init": "/usr/lib32/systemd/", "linux": "", "ro": "", "i915.modeset": "1"}},
+
+ {input: []string{},
+ want: map[string]string{}},
+ } {
+ got := getKclMap(tc.input)
+ for param, value := range tc.want {
+ if value != got[param] {
+ t.Fatalf("getKclMap expected:\n%v\ngot:\n%v", tc.want, got)
+ }
+ }
+ }
+}
diff --git a/src/cmd/cos_image_analyzer/internal/binary/info.go b/src/cmd/cos_image_analyzer/internal/binary/info.go
new file mode 100644
index 0000000..e5a350a
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/binary/info.go
@@ -0,0 +1,147 @@
+package binary
+
+import (
+ "bufio"
+ "errors"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/input"
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/utilities"
+)
+
+const cosGCSBucket = "cos-tools"
+const kernelHeaderGCSObject = "kernel-headers.tgz"
+const pathToKernelConfigs = "usr/src/linux-headers-4.19.112+/.config"
+
+const pathToKernelCommandLine = "efi/boot/grub.cfg" // Located in partition 12 EFI
+const kclImageName = "verified image A"
+const startOfHashingKCL = "dm="
+
+const pathToSysctlSettings = "/etc/sysctl.d/00-sysctl.conf" // Located in partition 3 Root-A
+
+// getPartitionStructure returns the partition structure of .raw file
+func getPartitionStructure(image *input.ImageInfo) error {
+ if image.TempDir == "" {
+ return nil
+ }
+
+ out, err := exec.Command("sudo", "sgdisk", "-p", image.DiskFile).Output()
+ if err != nil {
+ return fmt.Errorf("failed to call sgdisk -p %v: %v", image.DiskFile, err)
+ }
+
+ partitionFile := filepath.Join(image.TempDir, "partitions.txt")
+ if err := utilities.WriteToNewFile(partitionFile, string(out[:])); err != nil {
+ return fmt.Errorf("failed create file %v and write %v: %v", partitionFile, string(out[:]), err)
+ }
+ image.PartitionFile = partitionFile
+ return nil
+}
+
+// getKernelConfigs downloads the kernel configs for a build from GCS and stores
+// it into the image's temporary directory
+func getKernelConfigs(image *input.ImageInfo) error {
+ gcsObject := filepath.Join(image.BuildID, kernelHeaderGCSObject)
+ tarFile, err := utilities.GcsDowndload(cosGCSBucket, gcsObject, image.TempDir, kernelHeaderGCSObject, false)
+ if err != nil {
+ return fmt.Errorf("failed to download GCS object %v from bucket %v: %v", gcsObject, cosGCSBucket, err)
+ }
+
+ _, err = exec.Command("tar", "-xf", tarFile, "-C", image.TempDir).Output()
+ if err != nil {
+ return fmt.Errorf("failed to unzip %v into %v: %v", tarFile, image.TempDir, err)
+ }
+ image.KernelConfigsFile = filepath.Join(image.TempDir, pathToKernelConfigs)
+ return nil
+}
+
+// getKernelCommandLine gets the kernel command line from the image's partition 12 EFI
+// located in the /efi/boot/grub.cfg file
+func getKernelCommandLine(image *input.ImageInfo) error {
+ kclPath := filepath.Join(image.EFIPartition12, pathToKernelCommandLine)
+ kclFile, err := os.Open(kclPath)
+ if err != nil {
+ return fmt.Errorf("Failed to open file %v: %v", kclPath, err)
+ }
+ defer kclFile.Close()
+
+ foundKCL := false
+ scanner := bufio.NewScanner(kclFile)
+ for scanner.Scan() { // Scan file line by line for "verified Image A"
+ kcl := string(scanner.Text()[:])
+
+ if foundKCL {
+ if hashStart := strings.Index(kcl, startOfHashingKCL); hashStart >= 0 {
+ kcl = kcl[:hashStart] // Remove hash "dm='....'" from kcl
+ }
+ image.KernelCommandLine = strings.TrimSpace(kcl)
+ return nil
+ }
+ if strings.Contains(kcl, kclImageName) {
+ foundKCL = true
+ }
+ }
+
+ if scanner.Err() != nil {
+ return fmt.Errorf("Failed to scan file %v: %v", kclPath, scanner.Err())
+ }
+ return nil
+}
+
+// getSysctlSettings finds an image's Sysctrl settings file under
+// the /etc/sysctrl.d/00-sysctl.conf
+func getSysctlSettings(image *input.ImageInfo) error {
+ sysctlPath := filepath.Join(image.RootfsPartition3, pathToSysctlSettings)
+ image.SysctlSettingsFile = sysctlPath
+ return nil
+}
+
+// GetBinaryInfo finds relevant binary information for the COS image
+func GetBinaryInfo(image *input.ImageInfo, flagInfo *input.FlagInfo) error {
+ if image.TempDir == "" {
+ return nil
+ }
+
+ if image.RootfsPartition3 != "" { // Get Version and BuildID
+ osReleaseMap, err := utilities.ReadFileToMap(image.RootfsPartition3+etcOSRelease, "=")
+ if err != nil {
+ return fmt.Errorf("Failed to read /etc/os-release file in rootfs of image %v : %v", image.TempDir, err)
+ }
+ var ok bool
+ if image.Version, ok = osReleaseMap["VERSION"]; !ok {
+ return errors.New("Error: \"Version\" field not found in /etc/os-release file")
+ }
+ if image.BuildID, ok = osReleaseMap["BUILD_ID"]; !ok {
+ return errors.New("Error: \"Build_ID\" field not found in /etc/os-release file")
+ }
+ }
+
+ if utilities.InArray("Partition-structure", flagInfo.BinaryTypesSelected) { // Get partition structure from "sgdisk -p"
+ if err := getPartitionStructure(image); err != nil {
+ return fmt.Errorf("failed to get partition structure for image %v: %v", image.TempDir, err)
+ }
+ }
+
+ if utilities.InArray("Kernel-configs", flagInfo.BinaryTypesSelected) { // Get kernel configs from gs://cos-tools/BuildID/kernel-headers.tgz
+ if err := getKernelConfigs(image); err != nil {
+ return fmt.Errorf("failed to get kernel configs for image %v: %v", image.TempDir, err)
+ }
+ }
+
+ if utilities.InArray("Kernel-command-line", flagInfo.BinaryTypesSelected) { // Get kernel command line from partition 12 EFI (efi/boot/grub.cfg)
+ if err := getKernelCommandLine(image); err != nil {
+ return fmt.Errorf("failed to get the kernel command line for image %v: %v", image.TempDir, err)
+ }
+ }
+
+ if utilities.InArray("Sysctl-settings", flagInfo.BinaryTypesSelected) { // Get Sysctl settings from /etc/sysctl.d/00-sysctl.conf
+ if err := getSysctlSettings(image); err != nil {
+ return fmt.Errorf("failed to get Sysctl-settings for image %v: %v", image.TempDir, err)
+ }
+ }
+ return nil
+}
diff --git a/src/cmd/cos_image_analyzer/internal/binary/info_test.go b/src/cmd/cos_image_analyzer/internal/binary/info_test.go
new file mode 100644
index 0000000..e7ae5e0
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/binary/info_test.go
@@ -0,0 +1,44 @@
+package binary
+
+import (
+ "testing"
+
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/input"
+)
+
+// test GetBinaryInf function
+func TestGetBinaryInfo(t *testing.T) {
+ kclImage1 := `linux /syslinux/vmlinuz.A init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 dm_verity.error_behavior=3 dm_verity.max_bios=-1 dm_verity.dev_wait=1 i915.modeset=1 cros_efi root=/dev/dm-0`
+ for _, tc := range []struct {
+ image *input.ImageInfo
+ flagInfo *input.FlagInfo
+ want *input.ImageInfo
+ }{ // Version and BuildID
+ {image: &input.ImageInfo{TempDir: "../testdata/image1/", RootfsPartition3: "../testdata/image1/rootfs/"},
+ flagInfo: &input.FlagInfo{LocalPtr: true},
+ want: &input.ImageInfo{RootfsPartition3: "../testdata/image1", Version: "81", BuildID: "12871.119.0"}},
+ {image: &input.ImageInfo{},
+ flagInfo: &input.FlagInfo{LocalPtr: true},
+ want: &input.ImageInfo{}},
+ {image: &input.ImageInfo{TempDir: "../testdata/image2/", RootfsPartition3: "../testdata/image2/rootfs/"},
+ flagInfo: &input.FlagInfo{LocalPtr: true},
+ want: &input.ImageInfo{RootfsPartition3: "../testdata/image2/rootfs", Version: "77", BuildID: "12371.273.0"}},
+
+ // Kernel Command Line
+ {image: &input.ImageInfo{TempDir: "../testdata/image1", EFIPartition12: "../testdata/image1/efi/"},
+ flagInfo: &input.FlagInfo{BinaryTypesSelected: []string{"Kernel-command-line"}},
+ want: &input.ImageInfo{TempDir: "../testdata/image1", EFIPartition12: "../testdata/image1/efi/", KernelCommandLine: kclImage1}},
+ } {
+ GetBinaryInfo(tc.image, tc.flagInfo)
+
+ if tc.want.Version != tc.image.Version {
+ t.Fatalf("GetBinaryInfo expected: %v, got: %v", tc.want.Version, tc.image.Version)
+ }
+ if tc.want.BuildID != tc.image.BuildID {
+ t.Fatalf("GetBinaryInfo expected: %v, got: %v", tc.want.BuildID, tc.image.BuildID)
+ }
+ if tc.image.KernelCommandLine != tc.want.KernelCommandLine {
+ t.Fatalf("Diff kernel command line expected:$%v$, got:$%v$", tc.want.KernelCommandLine, tc.image.KernelCommandLine)
+ }
+ }
+}
diff --git a/src/cmd/cos_image_analyzer/internal/input/cleanup_api.go b/src/cmd/cos_image_analyzer/internal/input/cleanup_api.go
deleted file mode 100644
index 1d3bc50..0000000
--- a/src/cmd/cos_image_analyzer/internal/input/cleanup_api.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package input
-
-import (
- "os"
- "os/exec"
-)
-
-// Cleanup is called to remove a mounted directory and its loop device
-// (string) mountDir - Active mount directory ready to close
-// (string) loopDevice - Active loop device ready to close
-// Output: nil on success, else error
-func Cleanup(mountDir, loopDevice string) error {
- _, err := exec.Command("sudo", "umount", mountDir).Output()
- if err != nil {
- return err
- }
- _, err1 := exec.Command("sudo", "losetup", "-d", loopDevice).Output()
- if err1 != nil {
- return err1
- }
- os.Remove(mountDir)
- return nil
-}
diff --git a/src/cmd/cos_image_analyzer/internal/input/flaginfo.go b/src/cmd/cos_image_analyzer/internal/input/flaginfo.go
new file mode 100644
index 0000000..c978fd4
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/input/flaginfo.go
@@ -0,0 +1,49 @@
+package input
+
+// FlagInfo holds input preference from the user
+type FlagInfo struct {
+ // Args
+ Image1 string
+ Image2 string
+
+ // Input Types
+ LocalPtr bool
+ GcsPtr bool
+ CosCloudPtr bool
+
+ // Authentication
+ ProjectIDPtr string
+
+ // Binary
+ BinaryDiffPtr string
+ BinaryTypesSelected []string
+ // Package
+ PackageSelected bool
+ // Commit
+ CommitSelected bool
+ // Release Notes
+ ReleaseNotesSelected bool
+
+ // Verbosity of output
+ // If true, full Rootfs, Os-Config, and Stateful Partition output is shown.
+ // Else false (default), Rootfs and Stateful Partition directories listed on files
+ // pointed to by CompressRootfsFile and CompressStatefulFile respectively are compressed.
+ // For OS-configs difference, all /etc entries that are listed in CompressRootfsFile are ignored.
+ Verbose bool
+
+ // File used to compress directories in the output from Rootfs difference and
+ // for ignore entries under /etc for OS-Config difference
+ // (either user provided or default CompressRootfs.txt)
+ CompressRootfsFile string
+ // Slice of CompressRootfsFile
+ CompressRootfsSlice []string
+
+ // File used to compress directories in the output from Stateful-partition difference
+ // (either user provided or default CompressStateful.txt)
+ CompressStatefulFile string
+ // Slice of CompressRootfsFile
+ CompressStatefulSlice []string
+
+ // Output
+ OutputSelected string
+}
diff --git a/src/cmd/cos_image_analyzer/internal/input/gce_api.go b/src/cmd/cos_image_analyzer/internal/input/gce_api.go
deleted file mode 100644
index 9930b85..0000000
--- a/src/cmd/cos_image_analyzer/internal/input/gce_api.go
+++ /dev/null
@@ -1,97 +0,0 @@
-package input
-
-import (
- "bytes"
- "encoding/json"
-
- // "fmt"
- "io/ioutil"
- "log"
- "net/http"
- "path/filepath"
- "strings"
-)
-
-const timeOut = "7200"
-const imageFormat = "vmdk"
-const name = "gcr.io/compute-image-tools/gce_vm_image_export:release"
-
-type Steps struct {
- Args [6]string `json:"args"`
- Name string `json:"name"`
- Env [1]string `json:"env"`
-}
-
-type GcePayload struct {
- Timeout string `json:"timeout"`
- Steps [1]Steps `json:"steps"`
- Tags [2]string `json:"tags"`
-}
-
-// gceExport calls the cloud build REST api that exports a public compute
-// image to a specfic GCS bucket.
-// Input:
-// (string) projectID - project ID of the cloud project holding the image
-// (string) bucket - name of the GCS bucket holding the COS Image
-// (string) image - name of the source image to be exported
-// Output: None
-func gceExport(projectID, bucket, image string) error {
- // API Variables
- gceURL := "https://cloudbuild.googleapis.com/v1/projects/" + projectID + "/builds"
- destURI := "gs://" + bucket + "/" + image + "." + imageFormat
- args := [6]string{"-oauth=/usr/local/google/home/acueva/cos-googlesource/tools/src/cmd/cos_image_analyzer/internal/utilities/oauth.json", "-timeout=" + timeOut, "-source_image=" + image, "-client_id=api", "-format=" + imageFormat, "-destination_uri=" + destURI}
- env := [1]string{"BUILD_ID=$BUILD_ID"}
- tags := [2]string{"gce-daisy", "gce-daisy-image-export"}
-
- // Build API bodies
- steps := [1]Steps{Steps{Args: args, Name: name, Env: env}}
- payload := &GcePayload{
- Timeout: timeOut,
- Steps: steps,
- Tags: tags}
-
- requestBody, err := json.Marshal(payload)
- if err != nil {
- return err
- }
- log.Println(string(requestBody))
-
- resp, err := http.Post(gceURL, "application/json", bytes.NewBuffer(requestBody))
- if err != nil {
- return err
- }
- defer resp.Body.Close()
-
- body, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return err
- }
-
- log.Println(string(body))
- return nil
-}
-
-// GetCosImage calls the cloud build api to export a public COS image to a
-// a GCS bucket and then calls GetGcsImage() to download that image from GCS.
-// ADC is used for authorization.
-// Input:
-// (string) cosCloudPath - The "projectID/gcs-bucket/image" path of the
-// source image to be exported
-// Output:
-// (string) imageDir - Path to the mounted directory of the COS Image
-func GetCosImage(cosCloudPath string) (string, error) {
- spiltPath := strings.Split(cosCloudPath, "/")
- projectID, bucket, image := spiltPath[0], spiltPath[1], spiltPath[2]
-
- if err := gceExport(projectID, bucket, image); err != nil {
- return "", err
- }
-
- gcsPath := filepath.Join(bucket, image)
- imageDir, err := GetGcsImage(gcsPath, 1)
- if err != nil {
- return "", err
- }
-
- return imageDir, nil
-}
diff --git a/src/cmd/cos_image_analyzer/internal/input/gcs_api.go b/src/cmd/cos_image_analyzer/internal/input/gcs_api.go
deleted file mode 100644
index 0d56b99..0000000
--- a/src/cmd/cos_image_analyzer/internal/input/gcs_api.go
+++ /dev/null
@@ -1,156 +0,0 @@
-package input
-
-import (
- "bytes"
- "context"
- "io"
- "io/ioutil"
- "log"
- "os"
- "os/exec"
- "path/filepath"
- "strconv"
- "strings"
- "time"
-
- "cloud.google.com/go/storage"
-)
-
-const contextTimeOut = time.Second * 50
-
-// gcsDowndload calls the GCS client api to download a specifed object from
-// a GCS bucket. ADC is used for authorization
-// Input:
-// (io.Writier) w - Output destination for download info
-// (string) bucket - Name of the GCS bucket
-// (string) object - Name of the GCS object
-// (string) destDir - Destination for downloaded GCS object
-// Output:
-// (string) downloadedFile - Path to downloaded GCS object
-func gcsDowndload(w io.Writer, bucket, object, destDir string) (string, error) {
- // Call API to download GCS object into tempDir
- ctx := context.Background()
- client, err := storage.NewClient(ctx)
- if err != nil {
- return "", err
- }
- defer client.Close()
-
- ctx, cancel := context.WithTimeout(ctx, contextTimeOut)
- defer cancel()
-
- rc, err := client.Bucket(bucket).Object(object).NewReader(ctx)
- if err != nil {
- return "", err
- }
- defer rc.Close()
-
- data, err := ioutil.ReadAll(rc)
- if err != nil {
- return "", err
- }
-
- log.Print(log.New(w, "Blob "+object+" downloaded.\n", log.Ldate|log.Ltime|log.Lshortfile))
-
- downloadedFile := filepath.Join(destDir, object)
- if err := ioutil.WriteFile(downloadedFile, data, 0666); err != nil {
- return "", err
- }
- return downloadedFile, nil
-}
-
-// getPartitionStart finds the start partition offset of the disk
-// Input:
-// (string) diskFile - Name of DOS/MBR file (ex: disk.raw)
-// (string) parition - The parition number you are pulling the offset from
-// Output:
-// (int) start - The start of the partition on the disk
-func getPartitionStart(partition, diskRaw string) (int, error) {
- //create command
- cmd1 := exec.Command("fdisk", "-l", diskRaw)
- cmd2 := exec.Command("grep", "disk.raw"+partition)
-
- reader, writer := io.Pipe()
- var buf bytes.Buffer
-
- cmd1.Stdout = writer
- cmd2.Stdin = reader
- cmd2.Stdout = &buf
-
- cmd1.Start()
- cmd2.Start()
- cmd1.Wait()
- writer.Close()
- cmd2.Wait()
- reader.Close()
-
- words := strings.Fields(buf.String())
- start, err := strconv.Atoi(words[1])
- if err != nil {
- return -1, err
- }
-
- return start, nil
-}
-
-// mountDisk finds a free loop device and mounts a DOS/MBR disk file
-// Input:
-// (string) diskFile - Name of DOS/MBR file (ex: disk.raw)
-// (string) mountDir - Mount Destiination
-// Output: nil on success, else error
-func mountDisk(diskFile, mountDir string, flag int) error {
- sectorSize := 512
- startOfPartition, err := getPartitionStart("3", diskFile)
- if err != nil {
- return err
- }
- offset := strconv.Itoa(sectorSize * startOfPartition)
- out, err := exec.Command("sudo", "losetup", "--show", "-fP", diskFile).Output()
- if err != nil {
- return err
- }
- _, err1 := exec.Command("sudo", "mount", "-o", "ro,loop,offset="+offset, string(out[:len(out)-1]), mountDir).Output()
- if err1 != nil {
- return err1
- }
-
- return nil
-}
-
-// GetGcsImage calls the GCS client api that downloads a specifed object from
-// a GCS bucket and unzips its contents. ADC is used for authorization
-// Input:
-// (string) gcsPath - GCS "bucket/object" path for COS Image (.tar.gz file)
-// Output:
-// (string) imageDir - Path to the mounted directory of the COS Image
-func GetGcsImage(gcsPath string, flag int) (string, error) {
- bucket := strings.Split(gcsPath, "/")[0]
- object := strings.Split(gcsPath, "/")[1]
-
- tempDir, err := ioutil.TempDir(".", "tempDir-"+object) // Removed at end
- if err != nil {
- return "", err
- }
-
- tarFile, err := gcsDowndload(os.Stdout, bucket, object, tempDir)
- if err != nil {
- return "", err
- }
-
- imageDir := filepath.Join(tempDir, "Image-"+object)
- if err = os.Mkdir(imageDir, 0700); err != nil {
- return "", err
- }
-
- _, err1 := exec.Command("tar", "-xzf", tarFile, "-C", imageDir).Output()
- if err1 != nil {
- return "", err1
- }
-
- diskRaw := filepath.Join(imageDir, "disk.raw")
- if err = mountDisk(diskRaw, imageDir, flag); err != nil {
- return "", err
- }
-
- return imageDir, nil
-}
diff --git a/src/cmd/cos_image_analyzer/internal/input/imageinfo.go b/src/cmd/cos_image_analyzer/internal/input/imageinfo.go
new file mode 100644
index 0000000..f4583cc
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/input/imageinfo.go
@@ -0,0 +1,306 @@
+package input
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/utilities"
+)
+
+const gcsObjFormat = ".tar.gz"
+const makeDirFilemode = 0700
+const timeOut = "7200s"
+const imageFormat = "vmdk"
+const name = "gcr.io/compute-image-tools/gce_vm_image_export:release"
+const pathToKernelConfigs = "usr/src/linux-headers-4.19.112+/.config"
+const pathToSysctlSettings = "/etc/sysctl.d/00-sysctl.conf" // Located in partition 3 Root-A
+
+// ImageInfo stores all relevant information on a COS image
+type ImageInfo struct {
+ // Input Overhead
+ TempDir string // Temporary directory holding the mounted image and disk file
+ DiskFile string // Path to the DOS/MBR disk partition file
+ StatePartition1 string // Path to mounted directory of partition #1, stateful partition
+ RootfsPartition3 string // Path to mounted directory of partition #3, Rootfs-A
+ EFIPartition12 string // Path to mounted directory of partition #12, EFI-System
+ LoopDevice1 string // Active loop device for mounted image
+ LoopDevice3 string // Active loop device for mounted image
+ LoopDevice12 string // Active loop device for mounted image
+
+ // Binary info
+ Version string // Major cos version
+ BuildID string // Minor cos version
+ PartitionFile string // Path to the file storing the disk partition structure from "sgdisk"
+ SysctlSettingsFile string // Path to the /etc/sysctrl.d/00-sysctl.conf file of an image
+ KernelCommandLine string // The kernel command line boot-time parameters stored in partition 12 efi/boot/grub.cfg
+ KernelConfigsFile string // Path to the ".config" file downloaded from GCS that holds a build's kernel configs
+}
+
+// Rename temporary directory and its contents once Version and BuildID are known
+func (image *ImageInfo) Rename(flagInfo *FlagInfo) error {
+ if image.Version != "" && image.BuildID != "" {
+ fullImageName := "cos-" + image.Version + "-" + image.BuildID
+ if err := os.Rename(image.TempDir, fullImageName); err != nil {
+ return fmt.Errorf("failed to rename directory %v to %v: %v", image.TempDir, fullImageName, err)
+ }
+ image.TempDir = fullImageName
+
+ if !flagInfo.LocalPtr {
+ image.DiskFile = filepath.Join(fullImageName, "disk.raw")
+ }
+ if image.StatePartition1 != "" {
+ image.StatePartition1 = filepath.Join(fullImageName, "stateful")
+ }
+ if image.RootfsPartition3 != "" {
+ image.RootfsPartition3 = filepath.Join(fullImageName, "rootfs")
+ }
+ if image.EFIPartition12 != "" {
+ image.EFIPartition12 = filepath.Join(fullImageName, "efi")
+ }
+ image.PartitionFile = filepath.Join(fullImageName, "partitions.txt")
+ image.KernelConfigsFile = filepath.Join(fullImageName, pathToKernelConfigs)
+ image.SysctlSettingsFile = filepath.Join(image.RootfsPartition3, pathToSysctlSettings)
+ }
+ return nil
+}
+
+// MountImage is an ImagInfo method that mounts partitions 1,3 and 12 of
+// the image into the temporary directory
+// Input:
+// (string) arr - List of binary types selected from the user
+// Output: nil on success, else error
+func (image *ImageInfo) MountImage(arr []string) error {
+ if image.TempDir == "" {
+ return nil
+ }
+ if utilities.InArray("Stateful-partition", arr) {
+ stateful := filepath.Join(image.TempDir, "stateful")
+ if err := os.Mkdir(stateful, makeDirFilemode); err != nil {
+ return fmt.Errorf("failed to create make directory %v: %v", stateful, err)
+ }
+ image.StatePartition1 = stateful
+
+ loopDevice1, err := utilities.MountDisk(image.DiskFile, image.StatePartition1, "1")
+ if err != nil {
+ return fmt.Errorf("Failed to mount %v's partition #1 onto %v: %v", image.DiskFile, image.StatePartition1, err)
+ }
+ image.LoopDevice1 = loopDevice1
+ }
+
+ if utilities.InArray("Version", arr) || utilities.InArray("BuildID", arr) || utilities.InArray("Rootfs", arr) || utilities.InArray("Sysctl-settings", arr) || utilities.InArray("OS-config", arr) || utilities.InArray("Kernel-configs", arr) {
+ rootfs := filepath.Join(image.TempDir, "rootfs")
+ if err := os.Mkdir(rootfs, makeDirFilemode); err != nil {
+ return fmt.Errorf("failed to create make directory %v: %v", rootfs, err)
+ }
+ image.RootfsPartition3 = rootfs
+
+ loopDevice3, err := utilities.MountDisk(image.DiskFile, image.RootfsPartition3, "3")
+ if err != nil {
+ return fmt.Errorf("Failed to mount %v's partition #3 onto %v: %v", image.DiskFile, image.RootfsPartition3, err)
+ }
+ image.LoopDevice3 = loopDevice3
+ }
+
+ if utilities.InArray("Kernel-command-line", arr) {
+ efi := filepath.Join(image.TempDir, "efi")
+ if err := os.Mkdir(efi, makeDirFilemode); err != nil {
+ return fmt.Errorf("failed to create make directory %v: %v", efi, err)
+ }
+ image.EFIPartition12 = efi
+
+ loopDevice12, err := utilities.MountDisk(image.DiskFile, image.EFIPartition12, "12")
+ if err != nil {
+ return fmt.Errorf("Failed to mount %v's partition #12 onto %v: %v", image.DiskFile, image.EFIPartition12, err)
+ }
+ image.LoopDevice12 = loopDevice12
+ }
+ return nil
+}
+
+// GetGcsImage is an ImagInfo method that calls the GCS client api to
+// download a COS image from a GCS bucket, unzips it, and mounts relevant
+// partitions. ADC is used for authorization
+// Input:
+// (string) gcsPath - GCS "bucket/object" path for stored COS Image (.tar.gz file)
+// Output: nil on success, else error
+func (image *ImageInfo) GetGcsImage(gcsPath string) error {
+ if gcsPath == "" {
+ return nil
+ }
+ var gcsBucket, gcsObject string
+ if startOfBucket := strings.Index(gcsPath, "gs://"); startOfBucket < len(gcsPath)-5 {
+ gcsPath = gcsPath[startOfBucket+5:]
+ } else {
+ printUsage()
+ return errors.New("Error: Argument " + gcsPath + " is not a valid gcs path \"gs://<bucket>/<object_path>.tar.gz\"")
+ }
+ if startOfObject := strings.Index(gcsPath, "/"); startOfObject > 0 && startOfObject < len(gcsPath)-1 {
+ gcsBucket = gcsPath[:startOfObject]
+ gcsObject = gcsPath[startOfObject+1:]
+ } else {
+ printUsage()
+ return errors.New("Error: Argument " + gcsPath + " is not a valid gcs path \"gs://<bucket>/<object_path>.tar.gz\"")
+ }
+
+ tempDir, err := ioutil.TempDir(".", "tempDir") // Removed at end
+ if err != nil {
+ return fmt.Errorf("failed to create temporary directory: %v", err)
+ }
+ image.TempDir = tempDir
+
+ tarFile, err := utilities.GcsDowndload(gcsBucket, gcsObject, image.TempDir, filepath.Base(gcsObject), true)
+ if err != nil {
+ return fmt.Errorf("failed to download GCS object %v from bucket %v: %v", gcsObject, gcsBucket, err)
+ }
+
+ _, err = exec.Command("tar", "-xzf", tarFile, "-C", image.TempDir).Output()
+ if err != nil {
+ return fmt.Errorf("failed to unzip %v into %v: %v", tarFile, image.TempDir, err)
+ }
+ image.DiskFile = filepath.Join(image.TempDir, "disk.raw")
+ return nil
+}
+
+// GetLocalImage is an ImageInfo method that creates a temporary directory
+// to loop device mount the disk.raw file stored on the local file system
+// Input:
+// (string) localPath - Local path to the disk.raw file
+// Output: nil on success, else error
+func (image *ImageInfo) GetLocalImage(localPath string) error {
+ if localPath == "" {
+ return nil
+ }
+ image.DiskFile = localPath
+
+ tempDir, err := ioutil.TempDir(".", "tempDir") // Removed at end
+ if err != nil {
+ return fmt.Errorf("failed to create temporary directory: %v", err)
+ }
+ image.TempDir = tempDir
+ return nil
+}
+
+// steps holds GCE payload meta data
+type steps struct {
+ Args [5]string `json:"args"`
+ Name string `json:"name"`
+ Env [1]string `json:"env"`
+}
+
+// gcePayload holds GCE's rest API payload
+type gcePayload struct {
+ Timeout string `json:"timeout"`
+ Steps [1]steps `json:"steps"`
+ Tags [2]string `json:"tags"`
+}
+
+// gceExport calls the cloud build REST api that exports a public compute
+// image to a specific GCS bucket.
+// Input:
+// (string) projectID - project ID of the cloud project holding the image
+// (string) bucket - name of the GCS bucket holding the COS Image
+// (string) image - name of the source image to be exported
+// Output: nil on success, else error
+func gceExport(projectID, bucket, image string) error {
+ // API Variables
+ gceURL := "https://cloudbuild.googleapis.com/v1/projects/" + projectID + "/builds"
+ destURI := "gs://" + bucket + "/" + image + "." + imageFormat
+ args := [5]string{"-timeout=" + timeOut, "-source_image=" + image, "-client_id=api", "-format=" + imageFormat, "-destination_uri=" + destURI}
+ env := [1]string{"BUILD_ID=$BUILD_ID"}
+ tags := [2]string{"gce-daisy", "gce-daisy-image-export"}
+
+ // Build API bodies
+ steps := [1]steps{{Args: args, Name: name, Env: env}}
+ payload := gcePayload{
+ Timeout: timeOut,
+ Steps: steps,
+ Tags: tags}
+
+ requestBody, err := json.Marshal(payload)
+ if err != nil {
+ return fmt.Errorf("failed to json marshal GCE payload: %v", err)
+ }
+ log.Println(string(requestBody))
+
+ resp, err := http.Post(gceURL, "application/json", bytes.NewBuffer(requestBody))
+ if err != nil {
+ return fmt.Errorf("failed to make POST request: %v", err)
+ }
+ defer resp.Body.Close()
+
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return fmt.Errorf("failed to read returned POST request: %v", err)
+ }
+
+ log.Println(string(body))
+ return nil
+}
+
+// GetCosImage calls the cloud build api to export a public COS image to a
+// a GCS bucket and then calls GetGcsImage() to download that image from GCS.
+// ADC is used for authorization.
+// Input:
+// (*ImageInfo) image - A struct that holds the relevent
+// CosCloudPath "bucket/image" and projectID for the stored COS Image
+// Output: nil on success, else error
+func (image *ImageInfo) GetCosImage(cosCloudPath, projectID string) error {
+ if cosCloudPath == "" {
+ return nil
+ }
+ cosArray := strings.Split(cosCloudPath, "/")
+ if len(cosArray) != 2 {
+ printUsage()
+ return errors.New("Error: Argument " + cosCloudPath + " is not a valid cos-cloud path (\"/\" separators)")
+ }
+ gcsBucket := cosArray[0]
+ publicCosImage := cosArray[1]
+ if err := gceExport(projectID, gcsBucket, publicCosImage); err != nil {
+ return fmt.Errorf("failed to export %v cos image to GCS bucket %v: %v", publicCosImage, gcsBucket, err)
+ }
+
+ gcsPath := filepath.Join(gcsBucket, publicCosImage, gcsObjFormat)
+ if err := image.GetGcsImage(gcsPath); err != nil {
+ return fmt.Errorf("failed to download image stored on GCS for %v: %v", gcsPath, err)
+ }
+ return nil
+}
+
+// Cleanup is a ImageInfo method that removes a mounted directory & loop device
+// Input:
+// (*ImageInfo) image - A struct that holds the relevent info to clean up
+// Output: nil on success, else error
+func (image *ImageInfo) Cleanup() error {
+ if image.TempDir == "" {
+ return nil
+ }
+ if image.LoopDevice1 != "" {
+ if err := utilities.Unmount(image.StatePartition1, image.LoopDevice1); err != nil {
+ return fmt.Errorf("failed to unmount mount directory %v and/or loop device %v: %v", image.StatePartition1, image.LoopDevice1, err)
+ }
+ }
+ if image.LoopDevice3 != "" {
+ if err := utilities.Unmount(image.RootfsPartition3, image.LoopDevice3); err != nil {
+ return fmt.Errorf("failed to unmount mount directory %v and/or loop device %v: %v", image.RootfsPartition3, image.LoopDevice3, err)
+ }
+ }
+ if image.LoopDevice12 != "" {
+ if err := utilities.Unmount(image.EFIPartition12, image.LoopDevice12); err != nil {
+ return fmt.Errorf("failed to unmount mount directory %v and/or loop device %v: %v", image.EFIPartition12, image.LoopDevice12, err)
+ }
+ }
+
+ if err := os.RemoveAll(image.TempDir); err != nil {
+ return fmt.Errorf("failed to delete directory %v: %v", image.TempDir, err)
+ }
+ return nil
+}
diff --git a/src/cmd/cos_image_analyzer/internal/input/parse.go b/src/cmd/cos_image_analyzer/internal/input/parse.go
new file mode 100644
index 0000000..11854e3
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/input/parse.go
@@ -0,0 +1,285 @@
+package input
+
+import (
+ "errors"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/utilities"
+)
+
+// BinaryDiffTypes is a list of all valid binary differnce types
+var BinaryDiffTypes = []string{"Version", "BuildID", "Rootfs", "Kernel-command-line", "Stateful-partition", "Partition-structure", "Sysctl-settings", "OS-config", "Kernel-configs"}
+
+// Default Rootfs entires that are overridden by the "compress-rootfs" flag
+var defaultCompressRootfs = []string{"/bin/", "/lib/modules/", "/lib64/", "/usr/libexec/", "/usr/bin/", "/usr/sbin/", "/usr/lib64/", "/usr/share/zoneinfo/", "/usr/share/git/", "/usr/lib/", "/sbin/", "/etc/ssh/", "/etc/os-release/", "/etc/package_list/"}
+
+// Default Stateful entires that are overridden by the "compress-stateful" flag
+var defaultCompressStateful = []string{"/var_overlay/db/"}
+
+// Custom usage function. See -h flag
+func printUsage() {
+ usageTemplate := `NAME
+ cos_image_analyzer - finds all meaningful differences of two COS Images (binary and package differences).
+ If only one image is passed in, its binary info and package info will be returned.
+
+SYNOPSIS
+ %s [-local] FILE-1 [FILE-2] (default true)
+ FILE - the local file path to the DOS/MBR boot sector file of your image (Ex: disk.raw)
+ Ex: %s image-cos-77-12371-273-0/disk.raw image-cos-81-12871-119-0/disk.raw
+
+ %s -local -binary=Sysctl-settings,OS-config -package=false image-cos-77-12371-273-0/disk.raw
+
+ %s -gcs GCS-PATH-1 [GCS-PATH-2]
+ GCS-PATH - the GCS "gs://bucket/object" path for the COS Image ("object" is type .tar.gz)
+ Ex: %s -gcs gs://my-bucket/cos-images/cos-77-12371-273-0.tar.gz gs://my-bucket/cos-images/cos-81-12871-119-0.tar.gz
+
+
+DESCRIPTION
+ Input Flags:
+ -local (default true, flag is optional)
+ input is one or two DOS/MBR disk file on the local filesystem. If the images are downloaded from
+ Google Cloud as a tarball, decompress the tarball first then pass the disk.raw file to the program.
+ -gcs
+ input is one or two objects stored on Google Cloud Storage of type (.tar.gz). This flag temporarily downloads,
+ unzips, and loop device mounts the images into this tool's directory.
+ To download images from Google Cloud Storage, you need to pass a service account credential to the program.
+ Folllow https://cloud.google.com/docs/authentication/production#create_service_account to create a service account and
+ download the service account key. Then point environment variable GOOGLE_APPLICATION_CREDENTIALS to the key file then
+ run the program.
+
+ Difference Flags:
+ -binary (string)
+ specify which type of binary difference to show. Types "Version", "BuildID", "Kernel-command-line",
+ "Partition-structure", "Sysctl-settings", and "Kernel-configs" are supported for one and two image. "Rootfs",
+ "Stateful-partition", and "OS-config" are only supported for two images. To list multiple types separate by
+ comma. To NOT list any binary difference, set flag to "false". (default all types)
+ -package
+ specify whether to show package difference. Shows addition/removal of packages and package version updates.
+ To NOT list any package difference, set flag to false. (default false)
+
+ Attribute Flags
+ -verbose
+ include flag to increase verbosity of Rootfs, Stateful-partition, and OS-config differences. See -compress-rootfs and
+ -compress-stateful flags descriptions for the directories that are compressed by default.
+ -compress-rootfs (string)
+ to customize which directories are compressed in a non-verbose Rootfs and OS-config difference output, provide a local
+ file path to a .txt file. Format of the file must be one root file path per line with an ending back slash and no commas.
+ By default the directory(s) that are compressed during a diff are /bin/, /lib/modules/, /lib64/, /usr/libexec/, /usr/bin/,
+ /usr/sbin/, /usr/lib64/, /usr/share/zoneinfo/, /usr/share/git/, /usr/lib/, /sbin/, /etc/ssh/, /etc/os-release/ and
+ /etc/package_list/.
+ -compress-stateful (string)
+ to customize which directories are compressed in a non-verbose Stateful-partition difference output, provide a local
+ file path to a .txt file. Format of file must be one root file path per line with no commas. By default the directory(s)
+ that are compressed during a diff are /var_overlay/db/.
+
+ Output Flags:
+ -output (string)
+ Specify format of output. Only "terminal" stdout or "json" object is supported. (default "terminal")
+
+OUTPUT
+ Based on the "-output" flag. Either "terminal" stdout or machine readable "json" format.
+
+NOTE
+ The root permission is needed for this program because it needs to mount images into your local filesystem to calculate difference.
+`
+ cmd := filepath.Base(os.Args[0])
+ usage := fmt.Sprintf(usageTemplate, cmd, cmd, cmd, cmd, cmd)
+ fmt.Printf("%s", usage)
+}
+
+// FlagErrorChecking validates command-line flags stored in the FlagInfo struct
+// Input:
+// (*FlagInfo) flagInfo - A struct that stores all flag input
+// Output: nil on success, else error
+func FlagErrorChecking(flagInfo *FlagInfo) error {
+ // Error Checking
+ if (flagInfo.LocalPtr && flagInfo.GcsPtr) || (flagInfo.LocalPtr && flagInfo.CosCloudPtr) || (flagInfo.CosCloudPtr && flagInfo.GcsPtr) {
+ return errors.New("Error: Only one input flag is allowed. Multiple appeared")
+ }
+
+ if !(flagInfo.GcsPtr) && !(flagInfo.CosCloudPtr) {
+ flagInfo.LocalPtr = true
+ }
+
+ if flagInfo.BinaryDiffPtr == "" {
+ flagInfo.BinaryTypesSelected = BinaryDiffTypes
+ } else {
+ binaryTypesSelected := strings.Split(flagInfo.BinaryDiffPtr, ",")
+ for _, elem := range binaryTypesSelected {
+ if utilities.InArray(elem, BinaryDiffTypes) {
+ flagInfo.BinaryTypesSelected = append(flagInfo.BinaryTypesSelected, elem)
+ } else if elem != "false" {
+ return errors.New("Error: Invalid option for \"-binary\" flag")
+ }
+ }
+ }
+ if flagInfo.CompressRootfsFile != "" {
+ if res := utilities.FileExists(flagInfo.CompressRootfsFile, "txt"); res == -1 {
+ return errors.New("Error: " + flagInfo.CompressRootfsFile + " file does not exist")
+ } else if res == 0 {
+ return errors.New("Error: " + flagInfo.CompressRootfsFile + " is not a \".txt\" file")
+ }
+ }
+ if flagInfo.CompressStatefulFile != "" {
+ if res := utilities.FileExists(flagInfo.CompressStatefulFile, "txt"); res == -1 {
+ return errors.New("Error: " + flagInfo.CompressStatefulFile + " file does not exist")
+ } else if res == 0 {
+ return errors.New("Error: " + flagInfo.CompressStatefulFile + " is not a \".txt\" file")
+ }
+ }
+
+ if flagInfo.OutputSelected != "terminal" && flagInfo.OutputSelected != "json" {
+ return errors.New("Error: \"-output\" flag must be ethier \"terminal\" or \"json\"")
+ }
+
+ if len(flag.Args()) < 1 || len(flag.Args()) > 2 {
+ return errors.New("Error: Input must be one or two arguments")
+ }
+
+ flagInfo.Image1 = flag.Arg(0)
+ if len(flag.Args()) == 2 {
+ if flag.Arg(0) == flag.Arg(1) {
+ return errors.New("Error: Identical image passed in. To analyze single image, pass in one argument")
+ }
+ flagInfo.Image2 = flag.Arg(1)
+ }
+
+ return nil
+}
+
+// ParseFlags reads and validates the flags from the command-line
+// Input: None (Command-line flags and args)
+// Output:
+// (*FlagInfo) flagInfo - A struct that holds input preference from the user
+func ParseFlags() (*FlagInfo, error) {
+ flagInfo := &FlagInfo{}
+
+ flag.Usage = printUsage
+ flag.BoolVar(&flagInfo.LocalPtr, "local", false, "See printUsage for description")
+ flag.BoolVar(&flagInfo.GcsPtr, "gcs", false, "")
+ flag.BoolVar(&flagInfo.CosCloudPtr, "cos-cloud", false, "")
+
+ flag.StringVar(&flagInfo.ProjectIDPtr, "projectID", "", "")
+
+ flag.StringVar(&flagInfo.BinaryDiffPtr, "binary", "", "")
+ flag.BoolVar(&flagInfo.PackageSelected, "package", false, "")
+ flag.BoolVar(&flagInfo.CommitSelected, "commit", true, "")
+ flag.BoolVar(&flagInfo.ReleaseNotesSelected, "release-notes", true, "")
+
+ flag.BoolVar(&flagInfo.Verbose, "verbose", false, "")
+ flag.StringVar(&flagInfo.CompressRootfsFile, "compress-rootfs", "", "")
+ flag.StringVar(&flagInfo.CompressStatefulFile, "compress-stateful", "", "")
+
+ flag.StringVar(&flagInfo.OutputSelected, "output", "terminal", "")
+ flag.Parse()
+
+ if err := FlagErrorChecking(flagInfo); err != nil {
+ printUsage()
+ return &FlagInfo{}, err
+ }
+
+ if flagInfo.CompressRootfsFile != "" { // Get CompressRootfsslice
+ compressRootsBytes, err := ioutil.ReadFile(flagInfo.CompressRootfsFile)
+ if err != nil {
+ return &FlagInfo{}, fmt.Errorf("failed to read compress-rootfs file %v: %v", flagInfo.CompressRootfsFile, err)
+ }
+ flagInfo.CompressRootfsSlice = strings.Split(string(compressRootsBytes), "\n")
+ } else {
+ flagInfo.CompressRootfsSlice = defaultCompressRootfs
+ }
+
+ if flagInfo.CompressStatefulFile != "" { // Get CompressStatefulFileSlice
+ compressedStatefulBytes, err := ioutil.ReadFile(flagInfo.CompressStatefulFile)
+ if err != nil {
+ return &FlagInfo{}, fmt.Errorf("failed to read compress-stateful file %v: %v", flagInfo.CompressStatefulFile, err)
+ }
+ flagInfo.CompressStatefulSlice = strings.Split(string(compressedStatefulBytes), "\n")
+ } else {
+ flagInfo.CompressStatefulSlice = defaultCompressStateful
+ }
+ return flagInfo, nil
+}
+
+// validateLocalImages ensures the two images are one or two unique boot files
+// Input:
+// (string) localPath1 - Local path to the first disk.raw file
+// (string) localPath2 - Local path to the second disk.raw file
+// Output: nil on success, else error
+func validateLocalImages(localPath1, localPath2 string) error {
+ if localPath2 == "" {
+ if res := utilities.FileExists(localPath1, "raw"); res == -1 {
+ return errors.New("Error: " + localPath1 + " file does not exist")
+ } else if res == 0 {
+ return errors.New("Error: " + localPath1 + " is not a \".raw\" file")
+ }
+ return nil
+ }
+
+ if res := utilities.FileExists(localPath2, "raw"); res == -1 {
+ return errors.New("Error: " + localPath2 + " file does not exist")
+ } else if res == 0 {
+ return errors.New("Error: " + localPath2 + " is not a \".raw\" file")
+ }
+
+ info1, _ := os.Stat(localPath1)
+ info2, _ := os.Stat(localPath2)
+ if os.SameFile(info1, info2) {
+ return errors.New("Error: Identical image passed in. To analyze single image, pass in one argument")
+ }
+ return nil
+}
+
+// GetImages reads in all the flags and handles the input based on its type.
+// Input:
+// (*FlagInfo) flagInfo - A struct that holds input preference from the user
+// Output:
+// (*ImageInfo) image1 - A struct that stores relevent info for image1
+// (*ImageInfo) image2 - A struct that stores relevent info for image2
+func GetImages(flagInfo *FlagInfo) (*ImageInfo, *ImageInfo, error) {
+ image1, image2 := &ImageInfo{}, &ImageInfo{}
+
+ // Input Selection
+ if flagInfo.GcsPtr {
+ gcsPath1, gcsPath2 := flagInfo.Image1, flagInfo.Image2
+
+ if err := image1.GetGcsImage(gcsPath1); err != nil {
+ return image1, image2, fmt.Errorf("failed to download image stored on GCS for %s: %v", gcsPath1, err)
+ }
+ if err := image2.GetGcsImage(gcsPath2); err != nil {
+ return image1, image2, fmt.Errorf("failed to download image stored on GCS for %s: %v", gcsPath2, err)
+ }
+ return image1, image2, nil
+ } else if flagInfo.CosCloudPtr {
+ if flagInfo.ProjectIDPtr == "" {
+ return image1, image2, errors.New("Error: COS-cloud input requires the \"projectID\" flag to be set")
+ }
+ cosCloudPath1, cosCloudPath2 := flagInfo.Image1, flagInfo.Image2
+
+ if err := image1.GetCosImage(cosCloudPath1, flagInfo.ProjectIDPtr); err != nil {
+ return image1, image2, fmt.Errorf("failed to get cos image for %s: %v", cosCloudPath1, err)
+ }
+ if err := image2.GetCosImage(cosCloudPath2, flagInfo.ProjectIDPtr); err != nil {
+ return image1, image2, fmt.Errorf("failed to get cos image for %s: %v", cosCloudPath2, err)
+ }
+ return image1, image2, nil
+ } else if flagInfo.LocalPtr {
+ localPath1, localPath2 := flagInfo.Image1, flagInfo.Image2
+
+ if err := validateLocalImages(localPath1, localPath2); err != nil {
+ return image1, image2, fmt.Errorf("failed to validate local images: %v", err)
+ }
+ if err := image1.GetLocalImage(localPath1); err != nil {
+ return image1, image2, fmt.Errorf("failed to get local image for %s: %v", localPath1, err)
+ }
+ if err := image2.GetLocalImage(localPath2); err != nil {
+ return image1, image2, fmt.Errorf("failed to get local image for %s: %v", localPath2, err)
+ }
+ return image1, image2, nil
+ }
+ return image1, image2, errors.New("Error: At least one flag needs to be true")
+}
diff --git a/src/cmd/cos_image_analyzer/internal/input/parse_input.go b/src/cmd/cos_image_analyzer/internal/input/parse_input.go
deleted file mode 100644
index 9e081d1..0000000
--- a/src/cmd/cos_image_analyzer/internal/input/parse_input.go
+++ /dev/null
@@ -1,94 +0,0 @@
-package input
-
-import (
- "errors"
- "flag"
- "fmt"
- "os"
-)
-
-// Custom usage function. See -h flag
-func printUsage() {
- usageTemplate := `NAME
-cos_image_analyzer - finds all meaningful differences of two COS Images
-(binary, package, commit, and release notes differences)
-
-SYNOPSIS
-%s [-local] DIRECTORY-1 DIRECTORY-2 (default true)
- DIRECTORY 1/2 - the local directory path to the root of the COS Image
-
-%s [-gcs] GCS-PATH-1 GCS-PATH-2
- GCS-PATH 1/2 - GCS "bucket/object" path for the COS Image (.tar.gz file)
- Ex: %s -gcs my-bucket/cos-77-12371-273-0.tar.gz my-bucket/cos-81-12871-119-0.tar.gz
-
-%s [-cos-cloud] COS-CLOUD-PATH-1 COS-CLOUD-PATH-2
- COS-CLOUD-PATH 1/2 - The "projectID/gcs-bucket/image" path of the source image to be exported
- Ex: %s -cos-cloud my-project/my-bucket/my-exported-image1 my-project/my-bucket/my-exported-image2
-
-DESCRIPTION
-`
- usage := fmt.Sprintf(usageTemplate, os.Args[0], os.Args[0], os.Args[0], os.Args[0], os.Args[0])
- fmt.Printf("%s", usage)
- flag.PrintDefaults()
- fmt.Println("\nOUTPUT\n(stdout) terminal output - All differences printed to the terminal")
-}
-
-// ParseInput handles the input based on its type and returns the root
-// directory path of both images to the start of the CosImageAnalyzer
-//
-// Input: None (reads command-line args)
-//
-// Output: (string) rootImg1 - The local filesystem path for COS image1
-// (string) rootImg2 - The local filesystem path for COS image2
-func ParseInput() (string, string, error) {
- // Flag Declaration
- flag.Usage = printUsage
- localPtr := flag.Bool("local", true, "input is two mounted images on local filesystem")
- gcsPtr := flag.Bool("gcs", false, "input is two objects stored on Google Cloud Storage")
- cosCloudPtr := flag.Bool("cos-cloud", false, "input is two public COS-cloud images")
- flag.Parse()
-
- if flag.NFlag() > 1 {
- printUsage()
- return "", "", errors.New("Error: Only one flag allowed")
- }
-
- // Input Selection
- if *gcsPtr {
- if len(flag.Args()) != 2 {
- printUsage()
- return "", "", errors.New("Error: GCS input requires two agruments")
- }
- rootImg1, err := GetGcsImage(flag.Args()[0], 1)
- if err != nil {
- return "", "", err
- }
- rootImg2, err := GetGcsImage(flag.Args()[1], 2)
- if err != nil {
- return "", "", err
- }
- return rootImg1, rootImg2, nil
- } else if *cosCloudPtr {
- if len(flag.Args()) != 2 {
- printUsage()
- return "", "", errors.New("Error: COS-cloud input requires two agruments")
- }
- rootImg1, err := GetCosImage(flag.Args()[0])
- if err != nil {
- return "", "", err
- }
- rootImg2, err := GetCosImage(flag.Args()[1])
- if err != nil {
- return "", "", err
- }
- return rootImg1, rootImg2, nil
- } else if *localPtr {
- if len(flag.Args()) != 2 {
- printUsage()
- return "", "", errors.New("Error: Local input requires two arguments")
- }
- return flag.Args()[0], flag.Args()[1], nil
- }
- printUsage()
- return "", "", errors.New("Error: At least one flag needs to be true")
-}
diff --git a/src/cmd/cos_image_analyzer/internal/input/parse_test.go b/src/cmd/cos_image_analyzer/internal/input/parse_test.go
new file mode 100644
index 0000000..90d59fa
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/input/parse_test.go
@@ -0,0 +1,59 @@
+package input
+
+import (
+ "testing"
+
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/utilities"
+)
+
+// test FlagErrorChecking function
+func TestFlagErrorChecking(t *testing.T) {
+ for _, tc := range []struct {
+ input *FlagInfo
+ want *FlagInfo
+ wantErr bool
+ }{
+ {input: &FlagInfo{Image1: "../testdata/DOESNOTEXIST.txt", Image2: ""},
+ want: &FlagInfo{},
+ wantErr: true},
+ {input: &FlagInfo{Image1: "../testdata/false.raw", Image2: ""},
+ want: &FlagInfo{},
+ wantErr: true},
+ {input: &FlagInfo{Image1: "", Image2: ""},
+ want: &FlagInfo{},
+ wantErr: true},
+ {input: &FlagInfo{Image1: "arg0", Image2: "", LocalPtr: true, BinaryTypesSelected: []string{"wrongType"}},
+ want: &FlagInfo{},
+ wantErr: true},
+ {input: &FlagInfo{Image1: "arg0", Image2: "", LocalPtr: true, GcsPtr: true, CosCloudPtr: false, BinaryTypesSelected: []string{"BuildID"}},
+ want: &FlagInfo{},
+ wantErr: true},
+ {input: &FlagInfo{Image1: "arg0", Image2: "", LocalPtr: true, GcsPtr: false, CosCloudPtr: false, OutputSelected: "notJsonOrTerminal"},
+ want: &FlagInfo{},
+ wantErr: true},
+ {input: &FlagInfo{Image1: "arg0", Image2: "", LocalPtr: false, GcsPtr: false, CosCloudPtr: false, OutputSelected: "notJsonOrTerminal", BinaryTypesSelected: []string{"BuildID"}},
+ want: &FlagInfo{Image1: "arg0", Image2: "", LocalPtr: true, GcsPtr: false, CosCloudPtr: false, OutputSelected: "notJsonOrTerminal", BinaryTypesSelected: []string{"BuildID"}},
+ wantErr: false},
+ } {
+ gotErr := FlagErrorChecking(tc.input)
+ if tc.wantErr && gotErr == nil {
+ t.Fatalf("expected error but none returned")
+ }
+ if gotErr != nil {
+ continue
+ }
+
+ if tc.input.LocalPtr != tc.want.LocalPtr {
+ t.Fatalf("expected: %v, got: %v", tc.want.LocalPtr, tc.input.LocalPtr)
+ }
+ if tc.input.GcsPtr != tc.want.GcsPtr {
+ t.Fatalf("expected: %v, got: %v", tc.want.GcsPtr, tc.input.GcsPtr)
+ }
+ if tc.input.CosCloudPtr != tc.want.CosCloudPtr {
+ t.Fatalf("expected: %v, got: %v", tc.want.CosCloudPtr, tc.input.CosCloudPtr)
+ }
+ if !utilities.EqualArrays(tc.input.BinaryTypesSelected, tc.want.BinaryTypesSelected) {
+ t.Fatalf("expected: %v, got: %v", tc.want.BinaryTypesSelected, tc.input.BinaryTypesSelected)
+ }
+ }
+}
diff --git a/src/cmd/cos_image_analyzer/internal/output/imagediff.go b/src/cmd/cos_image_analyzer/internal/output/imagediff.go
new file mode 100644
index 0000000..7dbca8a
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/output/imagediff.go
@@ -0,0 +1,79 @@
+package output
+
+import (
+ "encoding/json"
+ "fmt"
+
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/binary"
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/input"
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/packagediff"
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/utilities"
+)
+
+// ImageDiff stores all of the differences between the two images
+type ImageDiff struct {
+ BinaryDiff *binary.Differences
+ PackageDiff *packagediff.Differences
+}
+
+// Formater is a ImageDiff function that outputs the image differences based on the "-output" flag.
+// Either to the terminal (default) or to a stored json object
+// Input:
+// (string) image1 - Temp directory name of image1
+// (string) image2 - Temp directory name of image2
+// (*FlagInfo) flagInfo - A struct that holds input preference from the user
+// Output:
+// ([]string) diffstrings/jsonObjectStr - Based on "-output" flag, either formated string
+// for the terminal or a string json object
+func (imageDiff *ImageDiff) Formater(image1, image2 string, flagInfo *input.FlagInfo) (string, error) {
+ if flagInfo.OutputSelected == "terminal" {
+ binaryStrings := ""
+ binaryFunctions := map[string]func() string{
+ "Version": imageDiff.BinaryDiff.FormatVersionDiff,
+ "BuildID": imageDiff.BinaryDiff.FormatBuildIDDiff,
+ "Rootfs": imageDiff.BinaryDiff.FormatRootfsDiff,
+ "Stateful-partition": imageDiff.BinaryDiff.FormatStatefulDiff,
+ "OS-config": imageDiff.BinaryDiff.FormatOSConfigDiff,
+ "Partition-structure": imageDiff.BinaryDiff.FormatPartitionStructureDiff,
+ "Kernel-configs": imageDiff.BinaryDiff.FormatKernelConfigsDiff,
+ "Kernel-command-line": imageDiff.BinaryDiff.FormatKernelCommandLineDiff,
+ "Sysctl-settings": imageDiff.BinaryDiff.FormatSysctlSettingsDiff,
+ }
+ for _, diff := range input.BinaryDiffTypes {
+ if utilities.InArray(diff, flagInfo.BinaryTypesSelected) {
+ binaryStrings += binaryFunctions[diff]()
+ }
+ }
+
+ if len(binaryStrings) > 0 {
+ if flagInfo.Image2 == "" {
+ binaryStrings = "================= Binary Info =================\nImage: " + image1 + "\n" + binaryStrings
+ } else {
+ binaryStrings = "================= Binary Differences =================\nImages: " + image1 + " and " + image2 + "\n" + binaryStrings
+ }
+ }
+
+ packageStrings := imageDiff.PackageDiff.FormatPackageListDiff(image1, image2)
+ if len(packageStrings) > 0 {
+ if flagInfo.Image2 == "" {
+ packageStrings = "================= Package List =================\nImage: " + image1 + "\n" + packageStrings
+ } else {
+ packageStrings = "================= Package Differences =================\nImages: " + image1 + " and " + image2 + "\n" + packageStrings
+ }
+ }
+
+ diffStrings := binaryStrings + packageStrings
+ return diffStrings, nil
+ }
+ jsonObjectBytes, err := json.Marshal(imageDiff)
+ if err != nil {
+ return "", fmt.Errorf("failed to json marshal the image difference struct: %v", err)
+ }
+ jsonObjectStr := string(jsonObjectBytes[:])
+ return jsonObjectStr, nil
+}
+
+// Print is a ImageDiff method that prints out all image differences
+func (imageDiff *ImageDiff) Print(differences string) {
+ fmt.Print(differences)
+}
diff --git a/src/cmd/cos_image_analyzer/internal/packagediff/diff.go b/src/cmd/cos_image_analyzer/internal/packagediff/diff.go
new file mode 100644
index 0000000..e3d3f6e
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/packagediff/diff.go
@@ -0,0 +1,150 @@
+package packagediff
+
+import (
+ "fmt"
+
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/input"
+)
+
+// PkgDiff is used to hold package difference between the two images
+type PkgDiff struct {
+ category []string
+ name []string
+ version []string
+ revision []string
+ typeOFDiff string // "image1" or "image2" if package is unique to image1 or image2, "shared" if package is shared in both images
+}
+
+// Differences is an intermediate struct used to store package lists and differences
+type Differences struct {
+ PackageDiff []PkgDiff // If two images are passed in, this is a slice of all package differences
+ PackageList []Package // If only one image is passed in, return full package list
+}
+
+// searchPackageList determines whether a package name appears in a package list
+func searchPackageList(packageName string, packageList []Package) (Package, bool) {
+ for _, p := range packageList {
+ if p.Name == packageName {
+ return p, true
+ }
+ }
+ return Package{}, false
+}
+
+// packageListDiff calculates the package list difference the two images
+// Input:
+// ([]Package) packagesImage1 - Image1's package list
+// ([]Package) packagesImage2 - Image2's package list
+// (*FlagInfo) flagInfo - A struct that holds input preference from the user
+// Output: nil on success, else err
+func (d *Differences) packageListDiff(packagesImage1, packagesImage2 []Package, flagInfo *input.FlagInfo) error {
+ if flagInfo.Image2 != "" {
+ for _, p1 := range packagesImage1 {
+ pkdDiff := PkgDiff{}
+ p2, ok := searchPackageList(p1.Name, packagesImage2)
+ if !ok { // Unique package to image 1
+ pkdDiff.typeOFDiff = "image1"
+ pkdDiff.name = []string{p1.Name}
+ pkdDiff.category = []string{p1.Category}
+ pkdDiff.version = []string{p1.Version}
+ pkdDiff.revision = []string{p1.Revision}
+ d.PackageDiff = append(d.PackageDiff, pkdDiff)
+ } else { // Shared package to image1 and image2
+ if p1.Category != p2.Category {
+ pkdDiff.category = []string{p1.Category, p2.Category}
+ }
+ if p1.Version != p2.Version {
+ pkdDiff.version = []string{p1.Version, p2.Version}
+ }
+ if p1.Revision != p2.Revision {
+ pkdDiff.revision = []string{p1.Revision, p2.Revision}
+ }
+ if len(pkdDiff.category) == 2 || len(pkdDiff.version) == 2 || len(pkdDiff.revision) == 2 {
+ pkdDiff.typeOFDiff = "shared"
+ pkdDiff.name = []string{p1.Name, p2.Name}
+ d.PackageDiff = append(d.PackageDiff, pkdDiff)
+ }
+ }
+ }
+
+ for _, p2 := range packagesImage2 {
+ pkdDiff := PkgDiff{}
+ if _, ok := searchPackageList(p2.Name, packagesImage1); !ok { // Unique package to image2
+ pkdDiff.typeOFDiff = "image2"
+ pkdDiff.category = []string{p2.Category}
+ pkdDiff.name = []string{p2.Name}
+ pkdDiff.version = []string{p2.Version}
+ pkdDiff.revision = []string{p2.Revision}
+ d.PackageDiff = append(d.PackageDiff, pkdDiff)
+ }
+ }
+ } else {
+ d.PackageList = packagesImage1
+ }
+ return nil
+}
+
+// FormatPackageListDiff returns a formated string of the package list difference
+// (string) image1 - Temp directory name of image1
+// (string) image2 - Temp directory name of image2
+func (d *Differences) FormatPackageListDiff(image1, image2 string) string {
+ if len(d.PackageList) > 0 { // One image is passed in, return full package list
+ pkgList := ""
+ for _, p := range d.PackageList {
+ pkgStr := "Package " + p.Name + "\n" + "category: " + p.Category + "\n" + "version: " + p.Version + "\n" + "revision: " + p.Revision + "\n\n"
+ pkgList += pkgStr
+ }
+ return pkgList
+ } else if len(d.PackageDiff) > 0 { // Two images are passed in, compare based on Differences
+ pkgDiff := ""
+ for _, pd := range d.PackageDiff {
+ pkgStr := ""
+ if pd.typeOFDiff == "shared" { // Compare shared packages
+ if len(pd.category) == 2 {
+ pkgStr += "category:\n" + "< " + pd.category[0] + "\n" + "> " + pd.category[1] + "\n"
+ }
+ if len(pd.version) == 2 {
+ pkgStr += "version:\n" + "< " + pd.version[0] + "\n" + "> " + pd.version[1] + "\n"
+ }
+ if len(pd.revision) == 2 {
+ pkgStr += "revision:\n" + "< " + pd.revision[0] + "\n" + "> " + pd.revision[1] + "\n"
+ }
+ if pkgStr != "" && len(pd.name) == 2 {
+ pkgStr = "Package " + pd.name[0] + " in " + image1 + " and " + image2 + " differ\n" + pkgStr + "\n"
+ pkgDiff += pkgStr
+ }
+ } else { // Unique package, return all info
+ if len(pd.name) == 1 && len(pd.category) == 1 && len(pd.version) == 1 && len(pd.revision) == 1 {
+ if pd.typeOFDiff == "image1" {
+ pkgStr += "Only in " + image1
+ } else if pd.typeOFDiff == "image2" {
+ pkgStr += "Only in " + image2
+ }
+ pkgStr += ": " + pd.name[0] + "\ncategory: " + pd.category[0] + "\nversion: " + pd.version[0] + "\nrevision: " + pd.revision[0] + "\n\n"
+ pkgDiff += pkgStr
+ }
+ }
+ }
+ return pkgDiff
+ }
+ return ""
+}
+
+// Diff is a tool that finds all package differences of two COS images
+// (Category, Name, Version, Revision)
+// Input:
+// (*ImageInfo) image1 - A struct that will store package info for image1
+// (*ImageInfo) image2 - A struct that will store package info for image2
+// (*FlagInfo) flagInfo - A struct that holds input preference from the user
+// Output:
+// (*Differences) packageDiff - A struct that will store the package differences
+func Diff(packagesImage1, packagesImage2 []Package, flagInfo *input.FlagInfo) (*Differences, error) {
+ packageDiff := &Differences{}
+
+ if flagInfo.PackageSelected {
+ if err := packageDiff.packageListDiff(packagesImage1, packagesImage2, flagInfo); err != nil {
+ return packageDiff, fmt.Errorf("failed to take package list difference: %v", err)
+ }
+ }
+ return packageDiff, nil
+}
diff --git a/src/cmd/cos_image_analyzer/internal/packagediff/diff_test.go b/src/cmd/cos_image_analyzer/internal/packagediff/diff_test.go
new file mode 100644
index 0000000..93afb8f
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/packagediff/diff_test.go
@@ -0,0 +1,130 @@
+package packagediff
+
+import (
+ "testing"
+
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/input"
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/utilities"
+)
+
+// test searchPackageList function
+func TestSearchPackageList(t *testing.T) {
+ testPackageList1 := []Package{
+ {Category: "sys-boot", Name: "shim"},
+ {Category: "chromeos-launch", Name: "cloud-network-boot", Version: "1.0.0", Revision: "4"},
+ {Revision: "533"}}
+ testPackageList2 := []Package{}
+ for _, tc := range []struct {
+ packageName string
+ packageList []Package
+ wantPackage Package
+ wantOk bool
+ }{
+ {packageName: "shim", packageList: testPackageList1, wantPackage: Package{Category: "sys-boot", Name: "shim"}, wantOk: true},
+ {packageName: "sys-boot", packageList: testPackageList1, wantPackage: Package{}, wantOk: false},
+ {packageName: "", packageList: testPackageList1, wantPackage: Package{Revision: "533"}, wantOk: true},
+ {packageName: "cloud-network-boot", packageList: testPackageList2, wantPackage: Package{}, wantOk: false},
+ } {
+ gotPackage, gotOk := searchPackageList(tc.packageName, tc.packageList)
+ if tc.wantOk != gotOk {
+ t.Fatalf("searchPackageList call expected: %v, got: %v", tc.wantOk, gotOk)
+ }
+ if tc.wantPackage.Name != gotPackage.Name {
+ t.Fatalf("searchPackageList expected: %v, got: %v", tc.wantPackage, gotPackage)
+ }
+ if tc.wantPackage.Category != gotPackage.Category {
+ t.Fatalf("searchPackageList expected: %v, got: %v", tc.wantPackage, gotPackage)
+ }
+ if tc.wantPackage.Version != gotPackage.Version {
+ t.Fatalf("searchPackageList expected: %v, got: %v", tc.wantPackage, gotPackage)
+ }
+ if tc.wantPackage.Revision != gotPackage.Revision {
+ t.Fatalf("searchPackageList expected: %v, got: %v", tc.wantPackage, gotPackage)
+ }
+ }
+}
+
+// test Diff function
+func TestDiff(t *testing.T) {
+ testPackageList1 := []Package{
+ {Category: "sys-boot", Name: "shim", Version: "14.0.20180308", Revision: "4"},
+ {Category: "chromeos-launch", Name: "cloud-network-boot", Version: "1.0.0", Revision: "4"},
+ {Category: "sys-kernel", Name: "lakitu-kernel-4_19", Version: "4.20.127", Revision: "533"},
+ {Category: "sys-apps", Name: "findutils", Version: "4.9.10", Revision: "1"},
+ {Category: "app-emulation", Name: "runc", Version: "1.0.0_rc10", Revision: "1"}}
+ testPackageList2 := []Package{}
+ testPackageList3 := []Package{
+ {Category: "sys-boot", Name: "shim", Version: "14.0.20180308", Revision: "4"},
+ {Category: "chromeos-base", Name: "cloud-network-init", Version: "1.0.0", Revision: "4"},
+ {Category: "sys-kernel", Name: "lakitu-kernel-4_19", Version: "4.19.127", Revision: "535"},
+ {Category: "app-shells", Name: "dash", Version: "0.5.9.1", Revision: "7"}}
+
+ testPackageDiff := []PkgDiff{
+ {category: []string{"chromeos-launch"}, name: []string{"cloud-network-boot"}, version: []string{"1.0.0"}, revision: []string{"4"}, typeOFDiff: "image1"},
+ {name: []string{"lakitu-kernel-4_19", "lakitu-kernel-4_19"}, version: []string{"4.20.127", "4.19.127"}, revision: []string{"533", "535"}, typeOFDiff: "shared"},
+ {category: []string{"sys-apps"}, name: []string{"findutils"}, version: []string{"4.9.10"}, revision: []string{"1"}, typeOFDiff: "image1"},
+ {category: []string{"app-emulation"}, name: []string{"runc"}, version: []string{"1.0.0_rc10"}, revision: []string{"1"}, typeOFDiff: "image1"},
+ {category: []string{"chromeos-base"}, name: []string{"cloud-network-init"}, version: []string{"1.0.0"}, revision: []string{"4"}, typeOFDiff: "image2"},
+ {category: []string{"app-shells"}, name: []string{"dash"}, version: []string{"0.5.9.1"}, revision: []string{"7"}, typeOFDiff: "image2"}}
+ testPkgDiffOneImage := &Differences{PackageList: testPackageList1}
+ testPkgDiffTwoImages := &Differences{PackageDiff: testPackageDiff}
+
+ for _, tc := range []struct {
+ packagesImage1 []Package
+ packagesImage2 []Package
+ flagInfo *input.FlagInfo
+ want *Differences
+ }{ // One Image Test
+ {packagesImage1: testPackageList1,
+ packagesImage2: testPackageList2,
+ flagInfo: &input.FlagInfo{Image2: "", PackageSelected: true},
+ want: testPkgDiffOneImage},
+ // TWo Image Test
+ {packagesImage1: testPackageList1,
+ packagesImage2: testPackageList3,
+ flagInfo: &input.FlagInfo{Image2: "../testdata/image2", PackageSelected: true},
+ want: testPkgDiffTwoImages},
+ } {
+ got, _ := Diff(tc.packagesImage1, tc.packagesImage2, tc.flagInfo)
+
+ for _, pl := range tc.want.PackageList {
+ pg, ok := searchPackageList(pl.Name, got.PackageList)
+ if !ok {
+ t.Fatalf("GetPackageInfo expected: %v, got: %v", tc.want, got)
+ }
+ if pl.Name != pg.Name {
+ t.Fatalf("GetPackageInfo expected: %v, got: %v", tc.want, got)
+ }
+ if pl.Category != pg.Category {
+ t.Fatalf("GetPackageInfo expected: %v, got: %v", tc.want, got)
+ }
+ if pl.Version != pg.Version {
+ t.Fatalf("GetPackageInfo expected: %v, got: %v", tc.want, got)
+ }
+ if pl.Revision != pg.Revision {
+ t.Fatalf("GetPackageInfo expected: %v, got: %v", tc.want, got)
+ }
+ }
+
+ for _, pd := range tc.want.PackageDiff {
+ foundPkg := false
+ for _, pg := range got.PackageDiff {
+ if utilities.EqualArrays(pd.name, pg.name) {
+ if !utilities.EqualArrays(pd.category, pg.category) {
+ t.Fatalf("Package diff expected: %v, got: %v", tc.want, got)
+ }
+ if !utilities.EqualArrays(pd.version, pg.version) {
+ t.Fatalf("Package diff expected: %v, got: %v", tc.want, got)
+ }
+ if !utilities.EqualArrays(pd.revision, pg.revision) {
+ t.Fatalf("Package diff expected: %v, got: %v", tc.want, got)
+ }
+ foundPkg = true
+ }
+ }
+ if !foundPkg {
+ t.Fatalf("Package diff expected: %v, got: %v", tc.want, got)
+ }
+ }
+ }
+}
diff --git a/src/cmd/cos_image_analyzer/internal/packagediff/info.go b/src/cmd/cos_image_analyzer/internal/packagediff/info.go
new file mode 100644
index 0000000..d5f8e39
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/packagediff/info.go
@@ -0,0 +1,59 @@
+package packagediff
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "path/filepath"
+
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/input"
+)
+
+const pathToPackageList = "/etc/package_list"
+
+// Package is used to store individual package data parsed from the package list json file
+type Package struct {
+ Category string
+ Name string
+ Version string
+ Revision string
+}
+
+// InstalledPackages is used to store an image’s full package list parsed from the package list json file
+type InstalledPackages struct {
+ InstalledPackages []Package
+}
+
+// ****** NOTE ******
+// This function is a temporary implementation. Switch this out with the awaited cos-tools library function.
+// getInstalledPackages returns the package list for an image by parsing its /etc/package_list json file
+func getInstalledPackages(rootfs string) ([]Package, error) {
+ fullPath := filepath.Join(rootfs, pathToPackageList)
+ packageListBytes, err := ioutil.ReadFile(fullPath)
+ if err != nil {
+ return []Package{}, fmt.Errorf("failed to read package list file %v: %v", fullPath, err)
+ }
+ var IP InstalledPackages
+ if err := json.Unmarshal(packageListBytes, &IP); err != nil {
+ return []Package{}, fmt.Errorf("failed to parse json for package list file %v: %v", fullPath, err)
+ }
+ return IP.InstalledPackages, nil
+}
+
+// ******************
+
+// GetPackageInfo finds relevant package list information for the COS image
+func GetPackageInfo(image *input.ImageInfo, flagInfo *input.FlagInfo) ([]Package, error) {
+ if image.TempDir == "" {
+ return []Package{}, nil
+ }
+
+ if flagInfo.PackageSelected { // Get package list from /etc/package_list
+ packageList, err := getInstalledPackages(image.RootfsPartition3)
+ if err != nil {
+ return []Package{}, fmt.Errorf("failed to get package list from image %v: %v", image.TempDir, err)
+ }
+ return packageList, nil
+ }
+ return []Package{}, nil
+}
diff --git a/src/cmd/cos_image_analyzer/internal/packagediff/info_test.go b/src/cmd/cos_image_analyzer/internal/packagediff/info_test.go
new file mode 100644
index 0000000..aa47ba7
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/packagediff/info_test.go
@@ -0,0 +1,62 @@
+package packagediff
+
+import (
+ "testing"
+
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/input"
+)
+
+// test GetPackageInfo function
+func TestGetPackageInfo(t *testing.T) {
+ testPackageList1 := []Package{
+ {Category: "sys-boot", Name: "shim", Version: "14.0.20180308", Revision: "4"},
+ {Category: "chromeos-launch", Name: "cloud-network-boot", Version: "1.0.0", Revision: "4"},
+ {Category: "sys-kernel", Name: "lakitu-kernel-4_19", Version: "4.20.127", Revision: "533"},
+ {Category: "sys-apps", Name: "findutils", Version: "4.9.10", Revision: "1"},
+ {Category: "app-emulation", Name: "runc", Version: "1.0.0_rc10", Revision: "1"}}
+ testPackageList2 := []Package{}
+ testPackageList3 := []Package{
+ {Category: "sys-boot", Name: "shim", Version: "14.0.20180308", Revision: "4"},
+ {Category: "chromeos-base", Name: "cloud-network-init", Version: "1.0.0", Revision: "4"},
+ {Category: "sys-kernel", Name: "lakitu-kernel-4_19", Version: "4.19.127", Revision: "535"},
+ {Category: "app-shells", Name: "dash", Version: "0.5.9.1", Revision: "7"}}
+ for _, tc := range []struct {
+ image *input.ImageInfo
+ flagInfo *input.FlagInfo
+ want []Package
+ }{ // Test image1
+ {image: &input.ImageInfo{TempDir: "../testdata/image1/", RootfsPartition3: "../testdata/image1/rootfs/"},
+ flagInfo: &input.FlagInfo{PackageSelected: true},
+ want: testPackageList1},
+ // Test empty image
+ {image: &input.ImageInfo{},
+ flagInfo: &input.FlagInfo{PackageSelected: true},
+ want: testPackageList2},
+ // Test image2
+ {image: &input.ImageInfo{TempDir: "../testdata/image2/", RootfsPartition3: "../testdata/image2/rootfs/"},
+ flagInfo: &input.FlagInfo{PackageSelected: true},
+ want: testPackageList3},
+ } {
+ got, _ := GetPackageInfo(tc.image, tc.flagInfo)
+
+ for _, pw := range tc.want {
+ pg, ok := searchPackageList(pw.Name, got)
+ if !ok {
+ t.Fatalf("GetPackageInfo expected: %v, got: %v", tc.want, got)
+ }
+ if pw.Name != pg.Name {
+ t.Fatalf("GetPackageInfo expected: %v, got: %v", tc.want, got)
+ }
+ if pw.Category != pg.Category {
+ t.Fatalf("GetPackageInfo expected: %v, got: %v", tc.want, got)
+ }
+ if pw.Version != pg.Version {
+ t.Fatalf("GetPackageInfo expected: %v, got: %v", tc.want, got)
+ }
+ if pw.Revision != pg.Revision {
+ t.Fatalf("GetPackageInfo expected: %v, got: %v", tc.want, got)
+ }
+
+ }
+ }
+}
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/blank.txt b/src/cmd/cos_image_analyzer/internal/testdata/blank.txt
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/blank.txt
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/false.raw b/src/cmd/cos_image_analyzer/internal/testdata/false.raw
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/false.raw
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/efi/efi/boot/grub.cfg b/src/cmd/cos_image_analyzer/internal/testdata/image1/efi/efi/boot/grub.cfg
new file mode 100644
index 0000000..6fba5a3
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/efi/efi/boot/grub.cfg
@@ -0,0 +1,35 @@
+defaultA=2
+defaultB=3
+gptpriority $grubdisk 2 prioA
+gptpriority $grubdisk 4 prioB
+
+if [ $prioA -lt $prioB ]; then
+ set default=$defaultB
+else
+ set default=$defaultA
+fi
+
+set timeout=0
+
+# NOTE: These magic grub variables are a Chrome OS hack. They are not portable.
+
+menuentry "local image A" {
+ linux /syslinux/vmlinuz.A init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 i915.modeset=1 cros_efi root=PARTUUID=E5822204-E5B9-2848-8A90-37790091EA3E
+}
+
+menuentry "local image B" {
+ linux /syslinux/vmlinuz.B init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 i915.modeset=1 cros_efi root=PARTUUID=BA6C8ED3-DD99-F046-BF74-3E2CF0F06234
+}
+
+menuentry "verified image A" {
+ linux /syslinux/vmlinuz.A init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 dm_verity.error_behavior=3 dm_verity.max_bios=-1 dm_verity.dev_wait=1 i915.modeset=1 cros_efi root=/dev/dm-0 dm="1 vroot none ro 1,0 2539520 verity payload=PARTUUID=E5822204-E5B9-2848-8A90-37790091EA3E hashtree=PARTUUID=E5822204-E5B9-2848-8A90-37790091EA3E hashstart=2539520 alg=sha256 root_hexdigest=f24f966c2c8e2dab5caeffd2ca4c406f31d3a7f4ffb3bcf578bd96c535bc01be salt=096198c60913c02972a43985fc8bb97ceb5359e75389d7ce7e3238139dd2078b"
+}
+
+menuentry "verified image B" {
+ linux /syslinux/vmlinuz.B init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 dm_verity.error_behavior=3 dm_verity.max_bios=-1 dm_verity.dev_wait=1 i915.modeset=1 cros_efi root=/dev/dm-0 dm="1 vroot none ro 1,0 2539520 verity payload=PARTUUID=BA6C8ED3-DD99-F046-BF74-3E2CF0F06234 hashtree=PARTUUID=BA6C8ED3-DD99-F046-BF74-3E2CF0F06234 hashstart=2539520 alg=sha256 root_hexdigest=f24f966c2c8e2dab5caeffd2ca4c406f31d3a7f4ffb3bcf578bd96c535bc01be salt=096198c60913c02972a43985fc8bb97ceb5359e75389d7ce7e3238139dd2078b"
+}
+
+# FIXME: usb doesn't support verified boot for now
+menuentry "Alternate USB Boot" {
+ linux (hd0,3)/boot/vmlinuz init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 root=PARTUUID=E5822204-E5B9-2848-8A90-37790091EA3E i915.modeset=1 cros_efi
+}
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/partitions.txt b/src/cmd/cos_image_analyzer/internal/testdata/image1/partitions.txt
new file mode 100644
index 0000000..dbc708e
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/partitions.txt
@@ -0,0 +1,22 @@
+Disk /img_disks/cos_81_12871_119_disk/disk.raw: 20971520 sectors, 10.0 GiB
+Sector size (logical): 512 bytes
+Disk identifier (GUID): 0274E604-5DE3-5E4E-A4FD-F4D00FBBD7AA
+Partition table holds up to 128 entries
+Main partition table begins at sector 2 and ends at sector 33
+First usable sector is 34, last usable sector is 18874491
+Partitions will be aligned on 1-sector boundaries
+Total free space is 135145 sectors (66.0 MiB)
+
+Number Start (sector) End (sector) Size Code Name
+ 1 8704000 18874476 4.8 GiB 8300 STATE
+ 2 20480 53247 16.0 MiB 7F00 KERN-A
+ 3 4509696 8703999 2.0 GiB 7F01 ROOT-A
+ 4 53248 86015 16.0 MiB 7F00 KERN-B
+ 5 315392 4509695 2.0 GiB 7F01 ROOT-B
+ 6 16448 16448 512 bytes 7F00 KERN-C
+ 7 16449 16449 512 bytes 7F01 ROOT-C
+ 8 86016 118783 16.0 MiB 8300 OEM
+ 9 16450 16450 512 bytes 7F02 reserved
+ 10 16451 16451 512 bytes 7F02 reserved
+ 11 64 16447 8.0 MiB EF02 RWFW
+ 12 249856 315391 32.0 MiB EF00 EFI-SYSTEM
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/docker/credentials.txt b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/docker/credentials.txt
new file mode 100644
index 0000000..e2b7a3d
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/docker/credentials.txt
@@ -0,0 +1,2 @@
+Name: docker.10.2.4
+job: makes micro kernels
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/docker/util/docker.txt b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/docker/util/docker.txt
new file mode 100644
index 0000000..3ad3b6d
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/docker/util/docker.txt
@@ -0,0 +1 @@
+docker configurations for image 1
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/docker/util/lib32/lib32.txt b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/docker/util/lib32/lib32.txt
new file mode 100644
index 0000000..0433c78
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/docker/util/lib32/lib32.txt
@@ -0,0 +1 @@
+32 bit library for docker
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/os-release b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/os-release
new file mode 100644
index 0000000..d4a89d7
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/os-release
@@ -0,0 +1,11 @@
+BUILD_ID=12871.119.0
+NAME="Container-Optimized OS"
+KERNEL_COMMIT_ID=fa84f12c6d738af9486e69a006a57df923f9476a
+GOOGLE_CRASH_ID=Lakitu
+VERSION_ID=81
+BUG_REPORT_URL="https://cloud.google.com/container-optimized-os/docs/resources/support-policy#contact_us"
+PRETTY_NAME="Container-Optimized OS from Google"
+VERSION=81
+GOOGLE_METRICS_PRODUCT_ID=26
+HOME_URL="https://cloud.google.com/container-optimized-os/docs"
+ID=cos
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/package_list b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/package_list
new file mode 100644
index 0000000..a78d074
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/package_list
@@ -0,0 +1,34 @@
+{
+ "InstalledPackages": [
+ {
+ "Category": "sys-boot",
+ "Name": "shim",
+ "Version": "14.0.20180308",
+ "Revision": "4"
+ },
+ {
+ "Category": "chromeos-launch",
+ "Name": "cloud-network-boot",
+ "Version": "1.0.0",
+ "Revision": "4"
+ },
+ {
+ "Category": "sys-kernel",
+ "Name": "lakitu-kernel-4_19",
+ "Version": "4.20.127",
+ "Revision": "533"
+ },
+ {
+ "Category": "sys-apps",
+ "Name": "findutils",
+ "Version": "4.9.10",
+ "Revision": "1"
+ },
+ {
+ "Category": "app-emulation",
+ "Name": "runc",
+ "Version": "1.0.0_rc10",
+ "Revision": "1"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/sysctl.d/00-sysctl.conf b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/sysctl.d/00-sysctl.conf
new file mode 100644
index 0000000..27dfc84
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/etc/sysctl.d/00-sysctl.conf
@@ -0,0 +1,11 @@
+# /etc/sysctl.conf
+# Look in /proc/sys/ for all the things you can setup.
+#
+
+# Enables source route verification
+net.ipv4.conf.default.rp_filter = 1
+# Enable reverse path
+net.ipv4.conf.all.rp_filter = 1
+
+# Disable shrinking the cwnd when connection is idle
+net.ipv4.tcp_slow_start_after_idle = 0
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/lib64/python.txt b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/lib64/python.txt
new file mode 100644
index 0000000..392bd39
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/lib64/python.txt
@@ -0,0 +1 @@
+Image 1's python info
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/proc/security/access.conf b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/proc/security/access.conf
new file mode 100644
index 0000000..764baa6
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/proc/security/access.conf
@@ -0,0 +1 @@
+testing 123 can you hear me?
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/proc/security/configs b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/proc/security/configs
new file mode 100644
index 0000000..fb6d1d0
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/proc/security/configs
@@ -0,0 +1 @@
+These are not the configs you are looking for
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/usr/lib/usr-lib-image1 b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/usr/lib/usr-lib-image1
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/rootfs/usr/lib/usr-lib-image1
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/stateful/dev_image/image1_dev.txt b/src/cmd/cos_image_analyzer/internal/testdata/image1/stateful/dev_image/image1_dev.txt
new file mode 100644
index 0000000..acf25dc
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/stateful/dev_image/image1_dev.txt
@@ -0,0 +1 @@
+dev space for image1
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/stateful/lost+found/Theseus.txt b/src/cmd/cos_image_analyzer/internal/testdata/image1/stateful/lost+found/Theseus.txt
new file mode 100644
index 0000000..7a19555
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/stateful/lost+found/Theseus.txt
@@ -0,0 +1 @@
+This is just test data, if you are lost check the README file
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/stateful/var_overlay/db/image1_data.txt b/src/cmd/cos_image_analyzer/internal/testdata/image1/stateful/var_overlay/db/image1_data.txt
new file mode 100644
index 0000000..0cc08a0
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/stateful/var_overlay/db/image1_data.txt
@@ -0,0 +1 @@
+writeable data for the user on image1
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image1/usr/src/linux-headers-4.19.112+/.config b/src/cmd/cos_image_analyzer/internal/testdata/image1/usr/src/linux-headers-4.19.112+/.config
new file mode 100644
index 0000000..bd58ab6
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image1/usr/src/linux-headers-4.19.112+/.config
@@ -0,0 +1,14 @@
+#
+# Compiler: Chromium OS 10.0_pre377782_p20200113-r10 clang version 10.0.0 (/var/cache/chromeos-cache/distfiles/host/egit-src/llvm-project 4e8231b5cf0f5f62c7a51a857e29f5be5cb55734)
+#
+CONFIG_GCC_VERSION=0
+CONFIG_CC_IS_CLANG=y
+CONFIG_CLANG_VERSION=100000
+
+#
+# General setup
+#
+CONFIG_INIT_ENV_ARG_LIMIT=32
+CONFIG_LOCALVERSION=""
+CONFIG_BUILD_SALT=""
+CONFIG_HAVE_KERNEL_GZIP=y
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/efi/efi/boot/grub.cfg b/src/cmd/cos_image_analyzer/internal/testdata/image2/efi/efi/boot/grub.cfg
new file mode 100755
index 0000000..58a9425
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/efi/efi/boot/grub.cfg
@@ -0,0 +1,35 @@
+defaultA=2
+defaultB=3
+gptpriority $grubdisk 2 prioA
+gptpriority $grubdisk 4 prioB
+
+if [ $prioA -lt $prioB ]; then
+ set default=$defaultB
+else
+ set default=$defaultA
+fi
+
+set timeout=0
+
+# NOTE: These magic grub variables are a Chrome OS hack. They are not portable.
+
+menuentry "local image A" {
+ linux /syslinux/vmlinuz.A init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 i915.modeset=1 cros_efi root=PARTUUID=D8242B83-5E09-2247-9220-A5581F2ADD0B
+}
+
+menuentry "local image B" {
+ linux /syslinux/vmlinuz.B init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 i915.modeset=1 cros_efi root=PARTUUID=D09026B0-6BC4-3C4A-A0D5-F7B8C67E96CC
+}
+
+menuentry "verified image A" {
+ linux /syslinux/vmlinuz.A init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 dm_verity.error_behavior=3 dm_verity.max_bios=-1 dm_verity.dev_wait=1 i915.modeset=1 cros_efi root=/dev/dm-0 dm="1 vroot none ro 1,0 4077568 verity payload=PARTUUID=D8242B83-5E09-2247-9220-A5581F2ADD0B hashtree=PARTUUID=D8242B83-5E09-2247-9220-A5581F2ADD0B hashstart=4077568 alg=sha256 root_hexdigest=1fcc4da1d3e2fa974479ac43cd3fdcc8127ede293888bfb7b0838fa795b4faeb salt=4b4c38200531b5c2f8bc29d1b7fe9f123da0c493ac086cc2dd5084888a61ce4b"
+}
+
+menuentry "verified image B" {
+ linux /syslinux/vmlinuz.B init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 dm_verity.error_behavior=3 dm_verity.max_bios=-1 dm_verity.dev_wait=1 i915.modeset=1 cros_efi root=/dev/dm-0 dm="1 vroot none ro 1,0 4077568 verity payload=PARTUUID=D09026B0-6BC4-3C4A-A0D5-F7B8C67E96CC hashtree=PARTUUID=D09026B0-6BC4-3C4A-A0D5-F7B8C67E96CC hashstart=4077568 alg=sha256 root_hexdigest=1fcc4da1d3e2fa974479ac43cd3fdcc8127ede293888bfb7b0838fa795b4faeb salt=4b4c38200531b5c2f8bc29d1b7fe9f123da0c493ac086cc2dd5084888a61ce4b"
+}
+
+# FIXME: usb doesn't support verified boot for now
+menuentry "Alternate USB Boot" {
+ linux (hd0,3)/boot/vmlinuz init=/usr/lib/systemd/systemd boot=local rootwait ro noresume noswap loglevel=7 noinitrd console=ttyS0 security=apparmor virtio_net.napi_tx=1 systemd.unified_cgroup_hierarchy=false systemd.legacy_systemd_cgroup_controller=false csm.disabled=1 root=PARTUUID=D8242B83-5E09-2247-9220-A5581F2ADD0B i915.modeset=1 cros_efi
+}
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/partitions.txt b/src/cmd/cos_image_analyzer/internal/testdata/image2/partitions.txt
new file mode 100644
index 0000000..ad5e7b3
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/partitions.txt
@@ -0,0 +1,22 @@
+Disk /img_disks/cos_77_12371_273_disk/disk.raw: 20971520 sectors, 10.0 GiB
+Sector size (logical): 512 bytes
+Disk identifier (GUID): AB9719F2-3174-4F46-8079-1CF470D2D9BC
+Partition table holds up to 128 entries
+Main partition table begins at sector 2 and ends at sector 33
+First usable sector is 34, last usable sector is 18874491
+Partitions will be aligned on 1-sector boundaries
+Total free space is 135145 sectors (66.0 MiB)
+
+Number Start (sector) End (sector) Size Code Name
+ 1 8704000 18874476 4.8 GiB 0700 STATE
+ 2 20480 53247 16.0 MiB 7F00 KERN-A
+ 3 4509696 8703999 2.0 GiB 7F01 ROOT-A
+ 4 53248 86015 16.0 MiB 7F00 KERN-B
+ 5 315392 4509695 2.0 GiB 7F01 ROOT-B
+ 6 16448 16448 512 bytes 7F00 KERN-C
+ 7 16449 16449 512 bytes 7F01 ROOT-C
+ 8 86016 118783 16.0 MiB 0700 OEM
+ 9 16450 16450 512 bytes 7F02 reserved
+ 10 16451 16451 512 bytes 7F02 reserved
+ 11 64 16447 8.0 MiB EF02 RWFW
+ 12 249856 315391 32.0 MiB EF00 EFI-SYSTEM
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/docker/credentials.txt b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/docker/credentials.txt
new file mode 100644
index 0000000..357d7c7
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/docker/credentials.txt
@@ -0,0 +1,2 @@
+Name: docker.10.2.1
+job: makes macro kernels
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/docker/util/lib64/lib64.txt b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/docker/util/lib64/lib64.txt
new file mode 100644
index 0000000..2936bb7
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/docker/util/lib64/lib64.txt
@@ -0,0 +1 @@
+64-bit library for docker
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/os-release b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/os-release
new file mode 100644
index 0000000..a6b134e
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/os-release
@@ -0,0 +1,11 @@
+BUILD_ID=12371.273.0
+NAME="Container-Optimized OS"
+KERNEL_COMMIT_ID=5d4ffd91281840f7a118143d77fbefb02e87943c
+GOOGLE_CRASH_ID=Lakitu
+VERSION_ID=77
+BUG_REPORT_URL="https://cloud.google.com/container-optimized-os/docs/resources/support-policy#contact_us"
+PRETTY_NAME="Container-Optimized OS from Google"
+VERSION=77
+GOOGLE_METRICS_PRODUCT_ID=26
+HOME_URL="https://cloud.google.com/container-optimized-os/docs"
+ID=cos
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/package_list b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/package_list
new file mode 100644
index 0000000..261d586
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/package_list
@@ -0,0 +1,28 @@
+{
+ "InstalledPackages": [
+ {
+ "Category": "sys-boot",
+ "Name": "shim",
+ "Version": "14.0.20180308",
+ "Revision": "4"
+ },
+ {
+ "Category": "chromeos-base",
+ "Name": "cloud-network-init",
+ "Version": "1.0.0",
+ "Revision": "4"
+ },
+ {
+ "Category": "sys-kernel",
+ "Name": "lakitu-kernel-4_19",
+ "Version": "4.19.127",
+ "Revision": "535"
+ },
+ {
+ "Category": "app-shells",
+ "Name": "dash",
+ "Version": "0.5.9.1",
+ "Revision": "7"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/sysctl.d/00-sysctl.conf b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/sysctl.d/00-sysctl.conf
new file mode 100644
index 0000000..2484c00
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/etc/sysctl.d/00-sysctl.conf
@@ -0,0 +1,14 @@
+# /etc/sysctl.conf
+# Look in /proc/sys/ for all the things you can setup.
+#
+
+# Enables source route verification
+net.ipv4.conf.default.rp_filter = 1
+# Enable reverse path
+net.ipv4.conf.all.rp_filter = 2
+
+# Disable shrinking the cwnd when connection is idle
+net.ipv4.tcp_slow_start_after_idle = 1
+
+# dumby variable
+net.ipv4.conf = 2
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/lib64/python.txt b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/lib64/python.txt
new file mode 100644
index 0000000..ce445c8
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/lib64/python.txt
@@ -0,0 +1 @@
+Image 2's python info
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/proc/security/access.conf b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/proc/security/access.conf
new file mode 100644
index 0000000..72ab259
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/proc/security/access.conf
@@ -0,0 +1 @@
+testing 456 can you hear me?
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/proc/security/configs b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/proc/security/configs
new file mode 100644
index 0000000..87be8a5
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/proc/security/configs
@@ -0,0 +1 @@
+These are the configs you are looking for
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/usr/lib/usr-lib-image2 b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/usr/lib/usr-lib-image2
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/rootfs/usr/lib/usr-lib-image2
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/stateful/dev_image/image2_dev.txt b/src/cmd/cos_image_analyzer/internal/testdata/image2/stateful/dev_image/image2_dev.txt
new file mode 100644
index 0000000..acf25dc
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/stateful/dev_image/image2_dev.txt
@@ -0,0 +1 @@
+dev space for image1
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/stateful/lost+found/Theseus.txt b/src/cmd/cos_image_analyzer/internal/testdata/image2/stateful/lost+found/Theseus.txt
new file mode 100644
index 0000000..7a19555
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/stateful/lost+found/Theseus.txt
@@ -0,0 +1 @@
+This is just test data, if you are lost check the README file
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/stateful/var_overlay/db/image2_data.txt b/src/cmd/cos_image_analyzer/internal/testdata/image2/stateful/var_overlay/db/image2_data.txt
new file mode 100644
index 0000000..979e8c3
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/stateful/var_overlay/db/image2_data.txt
@@ -0,0 +1 @@
+writeable data for the user on image2
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/image2/usr/src/linux-headers-4.19.112+/.config b/src/cmd/cos_image_analyzer/internal/testdata/image2/usr/src/linux-headers-4.19.112+/.config
new file mode 100644
index 0000000..75cd72a
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/testdata/image2/usr/src/linux-headers-4.19.112+/.config
@@ -0,0 +1,16 @@
+#
+# Compiler: Chromium OS 9.0_pre361749_p20190714-r4 clang version 9.0.0 (/var/cache/chromeos-cache/distfiles/host/egit-src/llvm-project c11de5eada2decd0a495ea02676b6f4838cd54fb) (based on LLVM 9.0.0svn)
+#
+CONFIG_GCC_VERSION=0
+CONFIG_CC_IS_CLANG=y
+CONFIG_CLANG_VERSION=90000
+
+#
+# General setup
+#
+CONFIG_INIT_ENV_ARG_LIMIT=32
+# CONFIG_COMPILE_TEST is not set
+CONFIG_LOCALVERSION=""
+# CONFIG_LOCALVERSION_AUTO is not set
+CONFIG_BUILD_SALT=""
+CONFIG_HAVE_KERNEL_GZIP=y
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/os-release-77 b/src/cmd/cos_image_analyzer/internal/testdata/os-release-77
deleted file mode 100644
index b21554c..0000000
--- a/src/cmd/cos_image_analyzer/internal/testdata/os-release-77
+++ /dev/null
@@ -1,2 +0,0 @@
-BUILD_ID=12371.273.0
-ID=cos
\ No newline at end of file
diff --git a/src/cmd/cos_image_analyzer/internal/testdata/os-release-81 b/src/cmd/cos_image_analyzer/internal/testdata/os-release-81
deleted file mode 100644
index 826d8f8..0000000
--- a/src/cmd/cos_image_analyzer/internal/testdata/os-release-81
+++ /dev/null
@@ -1,3 +0,0 @@
-BUILD_ID=12871.119.0
-VERSION=81
-ID=cos
diff --git a/src/cmd/cos_image_analyzer/internal/utilities/gcs_download.go b/src/cmd/cos_image_analyzer/internal/utilities/gcs_download.go
new file mode 100644
index 0000000..25a9e45
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/utilities/gcs_download.go
@@ -0,0 +1,71 @@
+package utilities
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "path/filepath"
+ "strconv"
+ "time"
+
+ "cloud.google.com/go/storage"
+ "google.golang.org/api/option"
+)
+
+const contextTimeOut = time.Second * 50
+const base10 = 10
+
+// GcsDowndload calls the GCS client api to download a specified object from
+// a GCS bucket.
+// Input:
+// (string) bucket - Name of the GCS bucket
+// (string) object - Name of the GCS object
+// (string) destDir - Destination for downloaded GCS object
+// (string) name - Name for the downloaded file
+// (bool) authenticate - Indicates whether the GCS client need to be authenticated.
+// Use unauthenticated client if you only wish to access public data.
+// Otherwise, ADC will be used for authorization.
+// Output:
+// (string) downloadedFile - Path to downloaded GCS object
+func GcsDowndload(bucket, object, destDir, name string, authenticate bool) (string, error) {
+ // Call API to download GCS object into tempDir
+ var client *storage.Client
+ var err error
+
+ ctx := context.Background()
+ if authenticate {
+ client, err = storage.NewClient(ctx)
+ } else {
+ client, err = storage.NewClient(ctx, option.WithoutAuthentication())
+ }
+ if err != nil {
+ return "", fmt.Errorf("failed to create new Google Cloud Go storage client: %v", err)
+ }
+ defer client.Close()
+
+ ctx, cancel := context.WithTimeout(ctx, contextTimeOut)
+ defer cancel()
+
+ rc, err := client.Bucket(bucket).Object(object).NewReader(ctx)
+ if err != nil {
+ return "", fmt.Errorf("failed to read GCS bucket: %v, and GCS object: %v : %v", bucket, object, err)
+ }
+ defer rc.Close()
+
+ downloadedFile, err := os.Create(filepath.Join(destDir, name))
+ if err != nil {
+ return "", fmt.Errorf("failed to create file %v/%v: %v", destDir, object, err)
+ }
+ defer downloadedFile.Close()
+
+ bytesDownloaded, err := io.Copy(downloadedFile, rc)
+ if err != nil {
+ return "", fmt.Errorf("failed to copy object into %v file: %v", downloadedFile, err)
+ }
+ bytesStr := strconv.FormatInt(bytesDownloaded, base10)
+
+ log.Print("GCS object: ", object, " downloaded from GCS bucket: ", bucket, ". Total bytes ", bytesStr)
+ return downloadedFile.Name(), nil
+}
diff --git a/src/cmd/cos_image_analyzer/internal/utilities/logic_helper.go b/src/cmd/cos_image_analyzer/internal/utilities/logic_helper.go
index bfdc0ea..a55efc9 100644
--- a/src/cmd/cos_image_analyzer/internal/utilities/logic_helper.go
+++ b/src/cmd/cos_image_analyzer/internal/utilities/logic_helper.go
@@ -1,9 +1,156 @@
package utilities
-// // Helper Function for error checking
-// func check(e error) error {
-// if e != nil {
-// return e
-// }
-// return nil
-// }
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "strconv"
+ "strings"
+)
+
+const sectorSize = 512
+
+// InArray determines if a string appears in a string array
+func InArray(val string, arr []string) bool {
+ for _, elem := range arr {
+ if elem == val {
+ return true
+ }
+ }
+ return false
+}
+
+//EqualArrays determines if two string arrays are equal
+func EqualArrays(arr1, arr2 []string) bool {
+ if len(arr1) != len(arr2) {
+ return false
+ }
+ for i, elem := range arr1 {
+ if arr2[i] != elem {
+ return false
+ }
+ }
+ return true
+}
+
+// FileExists determines if the path exists, and then if
+// the path points to a file of the desired type
+// Input:
+// (string) path - Local path to the file
+// (string) desiredType - The type of the file desired
+// Output: -1 if file doesn't exist, 0 if exists and is not
+// desiredType, and 1 if file exists and is desiredType
+func FileExists(path, desiredType string) int {
+ info, err := os.Stat(path)
+ if os.IsNotExist(err) || info.IsDir() {
+ return -1
+ }
+ fileName := strings.Split(info.Name(), ".")
+ fileType := fileName[len(fileName)-1]
+ if fileType != desiredType {
+ return 0
+ }
+ return 1
+}
+
+// WriteToNewFile creates a file and writes a string into it
+func WriteToNewFile(filename string, data string) error {
+ file, err := os.Create(filename)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+
+ _, err = io.WriteString(file, data)
+ if err != nil {
+ return err
+ }
+ return file.Sync()
+}
+
+// SliceToMapStr initializes a map with keys from input and empty strings as values
+func SliceToMapStr(input []string) map[string]string {
+ output := make(map[string]string)
+ for _, elem := range input {
+ output[elem] = ""
+ }
+ return output
+}
+
+// getPartitionStart finds the start partition offset of the disk
+// Input:
+// (string) diskFile - Name of DOS/MBR file (ex: disk.raw)
+// (string) partition - The partition number you are pulling the offset from
+// Output:
+// (int) start - The start of the partition on the disk
+func getPartitionStart(partition, diskRaw string) (int, error) {
+ //create command
+ cmd1 := exec.Command("fdisk", "-l", diskRaw)
+ cmd2 := exec.Command("grep", diskRaw+partition)
+
+ reader, writer := io.Pipe()
+ var buf bytes.Buffer
+
+ cmd1.Stdout = writer
+ cmd2.Stdin = reader
+ cmd2.Stdout = &buf
+
+ cmd1.Start()
+ cmd2.Start()
+ cmd1.Wait()
+ writer.Close()
+ cmd2.Wait()
+ reader.Close()
+
+ words := strings.Fields(buf.String())
+ if len(words) < 2 {
+ return -1, errors.New("Error: " + diskRaw + " is not a valid DOS/MBR boot sector file")
+ }
+ start, err := strconv.Atoi(words[1])
+ if err != nil {
+ return -1, fmt.Errorf("failed to convert Ascii %v to string: %v", words[1], err)
+ }
+
+ return start, nil
+}
+
+// MountDisk finds a free loop device and mounts a DOS/MBR disk file
+// Input:
+// (string) diskFile - Name of DOS/MBR file (ex: disk.raw)
+// (string) mountDir - Mount Destination
+// (string) partition - The partition number you are pulling the offset from
+// Output:
+// (string) loopDevice - Name of the loop device used to mount
+func MountDisk(diskFile, mountDir, partition string) (string, error) {
+ startOfPartition, err := getPartitionStart(partition, diskFile)
+ if err != nil {
+ return "", fmt.Errorf("failed to get start of partition #%v: %v", partition, err)
+ }
+ offset := strconv.Itoa(sectorSize * startOfPartition)
+
+ out, err := exec.Command("sudo", "losetup", "--show", "-fP", diskFile).Output()
+ if err != nil {
+ return "", fmt.Errorf("failed to create new loop device for %v: %v", diskFile, err)
+ }
+
+ loopDevice := string(out[:len(out)-1])
+ _, err = exec.Command("sudo", "mount", "-o", "ro,loop,offset="+offset, loopDevice, mountDir).Output()
+ if err != nil {
+ return "", fmt.Errorf("failed to mount loop device %v at %v: %v", loopDevice, mountDir, err)
+ }
+ return loopDevice, nil
+}
+
+// Unmount umounts a mounted directory and deletes its loop device
+func Unmount(mountedDirectory, loopDevice string) error {
+ if _, err := exec.Command("sudo", "umount", "-l", mountedDirectory).Output(); err != nil {
+ return fmt.Errorf("failed to umount directory %v: %v", mountedDirectory, err)
+ }
+ if _, err := exec.Command("sudo", "losetup", "-d", loopDevice).Output(); err != nil {
+ return fmt.Errorf("failed to delete loop device %v: %v", loopDevice, err)
+ }
+ return nil
+}
diff --git a/src/cmd/cos_image_analyzer/internal/utilities/logic_helper_test.go b/src/cmd/cos_image_analyzer/internal/utilities/logic_helper_test.go
new file mode 100644
index 0000000..98019dc
--- /dev/null
+++ b/src/cmd/cos_image_analyzer/internal/utilities/logic_helper_test.go
@@ -0,0 +1,95 @@
+package utilities
+
+import (
+ "testing"
+)
+
+// test TestInArray function
+func TestInArray(t *testing.T) {
+ type test struct {
+ testString string
+ testSlice []string
+ want bool
+ }
+
+ tests := []test{
+ {testString: "77", testSlice: []string{"77", "81"}, want: true},
+ {testString: "86", testSlice: []string{"77", "81"}, want: false},
+ {testString: "77", testSlice: []string{""}, want: false},
+ }
+
+ for _, tc := range tests {
+ got := InArray(tc.testString, tc.testSlice)
+ if tc.want != got {
+ t.Fatalf("InArray(%v, %v) call expected: %v, got: %v", tc.testString, tc.testSlice, tc.want, got)
+ }
+ }
+
+}
+
+// test TestEqualArrays function
+func TestEqualArrays(t *testing.T) {
+ type test struct {
+ testSlice1 []string
+ testSlice2 []string
+ want bool
+ }
+
+ tests := []test{
+ {testSlice1: []string{"77", "81"}, testSlice2: []string{"77", "81"}, want: true},
+ {testSlice1: []string{"77"}, testSlice2: []string{"77", "81"}, want: false},
+ {testSlice1: []string{}, testSlice2: []string{""}, want: false},
+ {testSlice1: []string{}, testSlice2: []string{}, want: true},
+ }
+
+ for _, tc := range tests {
+ got := EqualArrays(tc.testSlice1, tc.testSlice2)
+ if tc.want != got {
+ t.Fatalf("EqualArray(%v, %v) call expected: %v, got: %v", tc.testSlice1, tc.testSlice2, tc.want, got)
+ }
+ }
+}
+
+// test FileExists function
+func TestFileExists(t *testing.T) {
+ type test struct {
+ testFile string
+ testDesiredType string
+ want int
+ }
+
+ tests := []test{
+ {testFile: "../testdata/DOESNOTEXIST.txt", testDesiredType: "txt", want: -1},
+ {testFile: "../testdata/blank.txt", testDesiredType: "raw", want: 0},
+ {testFile: "../testdata/blank.txt", testDesiredType: "txt", want: 1},
+ }
+
+ for _, tc := range tests {
+ got := FileExists(tc.testFile, tc.testDesiredType)
+ if tc.want != got {
+ t.Fatalf("FileExits(%v, %v) call expected: %v, got: %v", tc.testFile, tc.testDesiredType, tc.want, got)
+ }
+ }
+}
+
+// test SliceToMapStr function
+func TestSliceToMapStr(t *testing.T) {
+ type test struct {
+ input []string
+ want map[string]string
+ }
+
+ tests := []test{
+ {input: []string{"a", "b", "c", "d"}, want: map[string]string{"a": "", "b": "", "c": "", "d": ""}},
+ {input: []string{}, want: map[string]string{}},
+ }
+
+ for _, tc := range tests {
+ got := SliceToMapStr(tc.input)
+ for k, v := range tc.want {
+ if v != got[k] {
+ t.Fatalf("SliceToMapStr call expected: %v, got: %v", tc.want, got)
+ }
+ }
+ }
+}
diff --git a/src/cmd/cos_image_analyzer/internal/utilities/map_helpers.go b/src/cmd/cos_image_analyzer/internal/utilities/map_helpers.go
index 06b1d39..b513fc5 100644
--- a/src/cmd/cos_image_analyzer/internal/utilities/map_helpers.go
+++ b/src/cmd/cos_image_analyzer/internal/utilities/map_helpers.go
@@ -12,14 +12,15 @@
// key: first word split by separator, value: rest of line after separator.
// Ex: Inputs: textLine: "NAME=Container-Optimized OS", sep: "="
// Outputs: map: {"NAME":"Container-Optimized OS"}
-//
-// Input: (string) filePath - The command-line path to the text file
-// (string) sep - The separator string for the key and value pairs
-// Output: (map[string]string) mapOfFile - The map of the read-in text file
+// Input:
+// (string) filePath - The command-line path to the text file
+// (string) sep - The separator string for the key and value pairs
+// Output:
+// (map[string]string) mapOfFile - The map of the read-in text file
func ReadFileToMap(filePath, sep string) (map[string]string, error) {
file, err := os.Open(filePath)
if err != nil {
- return map[string]string{}, err
+ return map[string]string{}, fmt.Errorf("Failed to open file %v: %v", filePath, err)
}
defer file.Close()
@@ -31,30 +32,30 @@
}
if scanner.Err() != nil {
- return map[string]string{}, scanner.Err()
+ return map[string]string{}, fmt.Errorf("Failed to scan file %v: %v", filePath, scanner.Err())
}
return mapOfFile, nil
}
// CmpMapValues is a helper function that compares a value shared by two maps
-// Input: (map[string]string) map1 - First map to be compared
-// (map[string]string) map2 - Second map to be compared
-// (string) key - The key of the value be compared in both maps
-//
-// Output: (stdout) terminal - If equal, print nothing. Else print difference
-// (int) result - -1 error, 0 for no difference, 1 for difference
+/// Input:
+// (map[string]string) map1 - First map to be compared
+// (map[string]string) map2 - Second map to be compared
+// (string) key - The key of the value be compared in both maps
+// Output:
+// (int) result - -1 for error, 0 for no difference, 1 for difference
func CmpMapValues(map1, map2 map[string]string, key string) (int, error) {
value1, ok1 := map1[key]
- value2, ok2 := map2[key]
+ value2, ok2 := "", true
+ if len(map2) != 0 {
+ value2, ok2 = map2[key]
+ }
if !ok1 || !ok2 { // Error Check: At least one key is not present
- return -1, errors.New("Error:" + key + "key not found in at least one of the maps")
+ return -1, errors.New("Error:" + key + " key not found in at least one of the maps")
}
if value1 != value2 {
- fmt.Println(key, "Difference")
- fmt.Println("< ", value1)
- fmt.Println("> ", value2)
return 1, nil
}
return 0, nil
diff --git a/src/cmd/cos_image_analyzer/internal/utilities/map_helpers_test.go b/src/cmd/cos_image_analyzer/internal/utilities/map_helpers_test.go
index 8e43712..64640f1 100644
--- a/src/cmd/cos_image_analyzer/internal/utilities/map_helpers_test.go
+++ b/src/cmd/cos_image_analyzer/internal/utilities/map_helpers_test.go
@@ -7,7 +7,7 @@
// test ReadFileToMap function
func TestReadFileToMap(t *testing.T) {
// test normal file
- testFile, sep := "../testdata/os-release-77", "="
+ testFile, sep := "../testdata/image2/rootfs/etc/os-release", "="
expectedMap := map[string]string{"BUILD_ID": "12371.273.0", "ID": "cos"}
resultMap, _ := ReadFileToMap(testFile, sep)
diff --git a/src/cmd/cos_image_analyzer/main.go b/src/cmd/cos_image_analyzer/main.go
index c66f3d9..293905b 100644
--- a/src/cmd/cos_image_analyzer/main.go
+++ b/src/cmd/cos_image_analyzer/main.go
@@ -1,26 +1,104 @@
// cos_Image_Analyzer finds all the meaningful differences of two COS Images
// (binary, package, commit, and release notes differences)
-//
-// Input: (string) rootImg1 - The path for COS image1
-// (string) rootImg2 - The path for COS image2
-// (int) inputFlag - 0-Local filesystem path to root directory,
-// 1-COS cloud names, 2-GCS object names
-//
-// Output: (stdout) terminal ouput - All differences printed to the terminal
+// Input:
+// (*ImageInfo) image1 - A struct that will store relevent info for image1
+// (*ImageInfo) image2 - A struct that will store relevent info for image2
+// (*FlagInfo) flagInfo - A struct that holds input preference from the user
+// Output:
+// Based on "-output" flag, either "terminal" stdout (default) or "json" obj
package main
import (
"fmt"
+ "log"
"os"
"runtime"
"cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/binary"
"cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/input"
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/output"
+ "cos.googlesource.com/cos/tools/src/cmd/cos_image_analyzer/internal/packagediff"
)
-func cosImageAnalyzer(img1Path, img2Path string) error {
- err := binary.BinaryDiff(img1Path, img2Path)
+func cosImageAnalyzer(image1, image2 *input.ImageInfo, flagInfo *input.FlagInfo) error {
+ imageDiff := &output.ImageDiff{}
+
+ err := *new(error)
+ if err := binary.GetBinaryInfo(image1, flagInfo); err != nil {
+ return fmt.Errorf("failed to get GetBinaryInfo from image %v: %v", flagInfo.Image1, err)
+ }
+ if err := binary.GetBinaryInfo(image2, flagInfo); err != nil {
+ return fmt.Errorf("failed to GetBinaryInfo from image %v: %v", flagInfo.Image2, err)
+ }
+ if err := image1.Rename(flagInfo); err != nil {
+ return fmt.Errorf("failed to rename image %v: %v", flagInfo.Image1, err)
+ }
+ if err := image2.Rename(flagInfo); err != nil {
+ return fmt.Errorf("failed to rename image %v: %v", flagInfo.Image2, err)
+ }
+
+ binaryDiff, err := binary.Diff(image1, image2, flagInfo)
if err != nil {
+ return fmt.Errorf("failed to get Binary Difference: %v", err)
+ }
+ imageDiff.BinaryDiff = binaryDiff
+
+ packageList1, err := packagediff.GetPackageInfo(image1, flagInfo)
+ if err != nil {
+ return fmt.Errorf("failed to get package info from image %v: %v", flagInfo.Image1, err)
+ }
+ packageList2, err := packagediff.GetPackageInfo(image2, flagInfo)
+ if err != nil {
+ return fmt.Errorf("failed to get package info from image %v: %v", flagInfo.Image2, err)
+ }
+ packageDiff, err := packagediff.Diff(packageList1, packageList2, flagInfo)
+ if err != nil {
+ return fmt.Errorf("failed to get package difference: %v", err)
+ }
+ imageDiff.PackageDiff = packageDiff
+
+ output, err := imageDiff.Formater(image1.TempDir, image2.TempDir, flagInfo)
+ if err != nil {
+ return fmt.Errorf("failed to format image difference: %v", err)
+ }
+ if flagInfo.OutputSelected == "terminal" {
+ imageDiff.Print(output)
+ } else {
+ fmt.Print(output)
+ }
+ return nil
+}
+
+// CallCosImageAnalyzer is wrapper that gets the images, calls cosImageAnalyzer, and cleans up
+func CallCosImageAnalyzer(image1, image2 *input.ImageInfo, flagInfo *input.FlagInfo) error {
+ if err := image1.MountImage(flagInfo.BinaryTypesSelected); err != nil {
+ return fmt.Errorf("failed to mount first image %v: %v", flagInfo.Image1, err)
+ }
+ if err := image2.MountImage(flagInfo.BinaryTypesSelected); err != nil {
+ return fmt.Errorf("failed to mount second image %v: %v", flagInfo.Image2, err)
+ }
+ if err := cosImageAnalyzer(image1, image2, flagInfo); err != nil {
+ return fmt.Errorf("failed to call cosImageAnalyzer: %v", err)
+ }
+ return nil
+}
+
+func analyze(flagInfo *input.FlagInfo) error {
+ var image1, image2 *input.ImageInfo
+ defer func() {
+ if err := image1.Cleanup(); err != nil {
+ log.Printf("failed to clean up image %v: %v", flagInfo.Image1, err)
+ }
+ if err := image2.Cleanup(); err != nil {
+ log.Printf("failed to clean up image %v: %v", flagInfo.Image2, err)
+ }
+ }()
+ var err error
+ image1, image2, err = input.GetImages(flagInfo)
+ if err != nil {
+ return fmt.Errorf("failed to get images: %v", err)
+ }
+ if err := CallCosImageAnalyzer(image1, image2, flagInfo); err != nil {
return err
}
return nil
@@ -29,20 +107,15 @@
func main() {
if runtime.GOOS != "linux" {
fmt.Printf("Error: This is a Linux tool, can not run on %s", runtime.GOOS)
- os.Exit(1)
}
- rootImg1, rootImg2, err := input.ParseInput()
+ flagInfo, err := input.ParseFlags()
if err != nil {
- fmt.Println(err)
+ log.Printf("failed to parse flags: %v\n", err)
os.Exit(1)
}
-
- err1 := cosImageAnalyzer(rootImg1, rootImg2)
- if err1 != nil {
- fmt.Println(err1)
+ if err := analyze(flagInfo); err != nil {
+ log.Printf("%v\n", err)
os.Exit(1)
}
- // Cleanup(rootImg1, loop1) Debating on a struct that holds this info
- // Cleanup(rootImg2, loop2)
-
+ os.Exit(0)
}
diff --git a/src/pkg/changelog/changelog.go b/src/pkg/changelog/changelog.go
old mode 100755
new mode 100644
index e984581..3b6c686
--- a/src/pkg/changelog/changelog.go
+++ b/src/pkg/changelog/changelog.go
@@ -47,9 +47,9 @@
manifestFileName string = "snapshot.xml"
// These constants are used for exponential increase in Gitiles request size.
- defaultPageSize int32 = 1000
- pageSizeGrowthMultiplier int32 = 5
- maxPageSize int32 = 10000
+ defaultPageSize int = 1000
+ pageSizeGrowthMultiplier int = 5
+ maxPageSize int = 10000
)
type repo struct {
@@ -66,16 +66,26 @@
}
type commitsResult struct {
- RepoURL string
- Commits []*Commit
- Err error
+ RepoURL string
+ Commits []*Commit
+ HasMoreCommits bool
+ Err error
}
type additionsResult struct {
- Additions map[string][]*Commit
+ Additions map[string]*RepoLog
Err error
}
+// limitPageSize will restrict a request page size to min of pageSize (which grows exponentially)
+// or remaining request size
+func limitPageSize(pageSize, requestedSize int) int {
+ if requestedSize == -1 || pageSize <= requestedSize {
+ return pageSize
+ }
+ return requestedSize
+}
+
func gerritClient(httpClient *http.Client, remoteURL string) (gitilesProto.GitilesClient, error) {
log.Debugf("Creating Gerrit client for remote url %s\n", remoteURL)
cl, err := gitilesApi.NewRESTClient(httpClient, remoteURL, true)
@@ -124,7 +134,9 @@
// Extract the "name", "remote", and "revision" attributes from each project tag.
// Some projects do not have a "remote" attribute.
// If this is the case, they should use the default remoteURL.
- remoteMap[""] = remoteMap[root.SelectElement("default").SelectAttr("remote").Value]
+ if root.SelectElement("default").SelectAttr("remote") != nil {
+ remoteMap[""] = remoteMap[root.SelectElement("default").SelectAttr("remote").Value]
+ }
repos := make(map[string]*repo)
for _, project := range root.SelectElements("project") {
repos[project.SelectAttr("name").Value] = &repo{
@@ -158,15 +170,17 @@
}
// commits get all commits that occur between committish and ancestor for a specific repo.
-func commits(client gitilesProto.GitilesClient, repo string, committish string, ancestor string, outputChan chan commitsResult) {
+func commits(client gitilesProto.GitilesClient, repo string, committish string, ancestor string, querySize int, outputChan chan commitsResult) {
log.Debugf("Fetching changelog for repo: %s on committish %s\n", repo, committish)
start := time.Now()
- pageSize := defaultPageSize
+
+ pageSize := limitPageSize(defaultPageSize, querySize)
+ querySize -= pageSize
request := gitilesProto.LogRequest{
Project: repo,
Committish: committish,
ExcludeAncestorsOf: ancestor,
- PageSize: pageSize,
+ PageSize: int32(pageSize),
}
response, err := client.Log(context.Background(), &request)
if err != nil {
@@ -174,6 +188,7 @@
repo, committish, ancestor, err)}
return
}
+
// No nextPageToken means there were less than <defaultPageSize> commits total.
// We can immediately return.
if response.NextPageToken == "" {
@@ -184,21 +199,23 @@
repo, committish, ancestor, err)}
return
}
- outputChan <- commitsResult{RepoURL: repo, Commits: parsedCommits}
+ outputChan <- commitsResult{RepoURL: repo, Commits: parsedCommits, HasMoreCommits: (response.NextPageToken != "")}
return
}
// Retrieve remaining commits using exponential increase in pageSize.
allCommits := response.Log
- for response.NextPageToken != "" {
+ for querySize > 0 && response.NextPageToken != "" {
if pageSize < maxPageSize {
pageSize *= pageSizeGrowthMultiplier
}
+ pageSize = limitPageSize(pageSize, querySize)
+ querySize -= pageSize
request := gitilesProto.LogRequest{
Project: repo,
Committish: committish,
ExcludeAncestorsOf: ancestor,
PageToken: response.NextPageToken,
- PageSize: pageSize,
+ PageSize: int32(pageSize),
}
response, err = client.Log(context.Background(), &request)
if err != nil {
@@ -215,14 +232,14 @@
repo, committish, ancestor, err)}
return
}
- outputChan <- commitsResult{RepoURL: repo, Commits: parsedCommits}
+ outputChan <- commitsResult{RepoURL: repo, Commits: parsedCommits, HasMoreCommits: (response.NextPageToken != "")}
}
// additions retrieves all commits that occured between 2 parsed manifest files for each repo.
// Returns a map of repo name -> list of commits.
-func additions(clients map[string]gitilesProto.GitilesClient, sourceRepos map[string]*repo, targetRepos map[string]*repo, outputChan chan additionsResult) {
+func additions(clients map[string]gitilesProto.GitilesClient, sourceRepos map[string]*repo, targetRepos map[string]*repo, querySize int, outputChan chan additionsResult) {
log.Debug("Retrieving commit additions")
- repoCommits := make(map[string][]*Commit)
+ repoCommits := make(map[string]*RepoLog)
commitsChan := make(chan commitsResult, len(targetRepos))
for repoURL, targetRepoInfo := range targetRepos {
cl := clients[targetRepoInfo.InstanceURL]
@@ -232,7 +249,7 @@
if sourceRepoInfo, ok := sourceRepos[repoURL]; ok {
ancestorCommittish = sourceRepoInfo.Committish
}
- go commits(cl, repoURL, targetRepoInfo.Committish, ancestorCommittish, commitsChan)
+ go commits(cl, repoURL, targetRepoInfo.Committish, ancestorCommittish, querySize, commitsChan)
}
for i := 0; i < len(targetRepos); i++ {
res := <-commitsChan
@@ -240,8 +257,17 @@
outputChan <- additionsResult{Err: res.Err}
return
}
+ sourceSHA := ""
+ if sha, ok := sourceRepos[res.RepoURL]; ok {
+ sourceSHA = sha.Committish
+ }
if len(res.Commits) > 0 {
- repoCommits[res.RepoURL] = res.Commits
+ repoCommits[res.RepoURL] = &RepoLog{
+ Commits: res.Commits,
+ HasMoreCommits: res.HasMoreCommits,
+ SourceSHA: sourceSHA,
+ TargetSHA: targetRepos[res.RepoURL].Committish,
+ }
}
}
outputChan <- additionsResult{Additions: repoCommits}
@@ -257,19 +283,26 @@
// a tag that links directly to snapshot.xml
// Ex. For /refs/tags/15049.0.0, the argument should be 15049.0.0
//
-// The host should be the GoB instance that Manifest files are hosted in
+// host should be the GoB instance that Manifest files are hosted in
// ex. "cos.googlesource.com"
//
-// The repo should be the repository that build manifest files
+// repo should be the repository that build manifest files
// are located, ex. "cos/manifest-snapshots"
//
+// querySize should be the number of commits that should be included in each
+// repository changelog. Specify as -1 to get all commits
+//
// Outputs two changelogs
// The first changelog contains new commits that were added to the target
// build starting from the source build number
//
// The second changelog contains all commits that are present in the source build
// but not present in the target build
-func Changelog(httpClient *http.Client, sourceBuildNum string, targetBuildNum string, host string, repo string) (map[string][]*Commit, map[string][]*Commit, error) {
+func Changelog(httpClient *http.Client, sourceBuildNum string, targetBuildNum string, host string, repo string, querySize int) (map[string]*RepoLog, map[string]*RepoLog, error) {
+ if httpClient == nil {
+ return nil, nil, errors.New("Changelog: httpClient should not be nil")
+ }
+
log.Infof("Retrieving changelog between %s and %s\n", sourceBuildNum, targetBuildNum)
clients := make(map[string]gitilesProto.GitilesClient)
@@ -302,16 +335,16 @@
addChan := make(chan additionsResult, 1)
missChan := make(chan additionsResult, 1)
- go additions(clients, sourceRepos, targetRepos, addChan)
- go additions(clients, targetRepos, sourceRepos, missChan)
- addRes := <-addChan
- if addRes.Err != nil {
- return nil, nil, fmt.Errorf("Changelog: failure when retrieving commit additions:\n%v", err)
- }
+ go additions(clients, sourceRepos, targetRepos, querySize, addChan)
+ go additions(clients, targetRepos, sourceRepos, querySize, missChan)
missRes := <-missChan
if missRes.Err != nil {
return nil, nil, fmt.Errorf("Changelog: failure when retrieving missed commits:\n%v", err)
}
+ addRes := <-addChan
+ if addRes.Err != nil {
+ return nil, nil, fmt.Errorf("Changelog: failure when retrieving commit additions:\n%v", err)
+ }
return addRes.Additions, missRes.Additions, nil
}
diff --git a/src/pkg/changelog/changelog_test.go b/src/pkg/changelog/changelog_test.go
index 6113444..86b6423 100644
--- a/src/pkg/changelog/changelog_test.go
+++ b/src/pkg/changelog/changelog_test.go
@@ -51,9 +51,9 @@
return true
}
-func mappingInLog(log map[string][]*Commit, check []string) bool {
+func repoListInLog(log map[string]*RepoLog, check []string) bool {
for _, check := range check {
- if log, ok := log[check]; !ok || len(log) == 0 {
+ if log, ok := log[check]; !ok || len(log.Commits) == 0 {
return false
}
}
@@ -64,57 +64,68 @@
httpClient, err := getHTTPClient()
// Test invalid source
- additions, misses, err := Changelog(httpClient, "15", "15043.0.0", cosInstance, defaultManifestRepo)
+ additions, removals, err := Changelog(httpClient, "15", "15043.0.0", cosInstance, defaultManifestRepo, -1)
if additions != nil {
t.Errorf("Changelog failed, expected nil additions, got %v", additions)
- } else if misses != nil {
- t.Errorf("Changelog failed, expected nil misses, got %v", misses)
+ } else if removals != nil {
+ t.Errorf("Changelog failed, expected nil removals, got %v", removals)
} else if err == nil {
t.Errorf("Changelog failed, expected error, got nil")
}
// Test invalid target
- additions, misses, err = Changelog(httpClient, "15043.0.0", "abx", cosInstance, defaultManifestRepo)
+ additions, removals, err = Changelog(httpClient, "15043.0.0", "abx", cosInstance, defaultManifestRepo, -1)
if additions != nil {
t.Errorf("Changelog failed, expected nil additions, got %v", additions)
- } else if misses != nil {
- t.Errorf("Changelog failed, expected nil misses, got %v", misses)
+ } else if removals != nil {
+ t.Errorf("Changelog failed, expected nil removals, got %v", removals)
} else if err == nil {
t.Errorf("Changelog failed, expected error, got nil")
}
// Test invalid instance
- additions, misses, err = Changelog(httpClient, "15036.0.0", "15041.0.0", "com", defaultManifestRepo)
+ additions, removals, err = Changelog(httpClient, "15036.0.0", "15041.0.0", "com", defaultManifestRepo, -1)
if additions != nil {
t.Errorf("Changelog failed, expected nil additions, got %v", additions)
- } else if misses != nil {
- t.Errorf("Changelog failed, expected nil misses, got %v", misses)
+ } else if removals != nil {
+ t.Errorf("Changelog failed, expected nil removals, got %v", removals)
} else if err == nil {
t.Errorf("Changelog failed, expected error, got nil")
}
// Test invalid manifest repo
- additions, misses, err = Changelog(httpClient, "15036.0.0", "15041.0.0", cosInstance, "cos/not-a-repo")
+ additions, removals, err = Changelog(httpClient, "15036.0.0", "15041.0.0", cosInstance, "cos/not-a-repo", -1)
if additions != nil {
t.Errorf("Changelog failed, expected nil additions, got %v", additions)
- } else if misses != nil {
- t.Errorf("Changelog failed, expected nil misses, got %v", misses)
+ } else if removals != nil {
+ t.Errorf("Changelog failed, expected nil removals, got %v", removals)
} else if err == nil {
t.Errorf("Changelog failed, expected error, got nil")
}
// Test build number higher than latest release
- additions, misses, err = Changelog(httpClient, "15036.0.0", "99999.0.0", cosInstance, defaultManifestRepo)
+ additions, removals, err = Changelog(httpClient, "15036.0.0", "99999.0.0", cosInstance, defaultManifestRepo, -1)
if additions != nil {
t.Errorf("Changelog failed, expected nil additions, got %v", additions)
- } else if misses != nil {
- t.Errorf("Changelog failed, expected nil misses, got %v", misses)
+ } else if removals != nil {
+ t.Errorf("Changelog failed, expected nil removals, got %v", removals)
} else if err == nil {
t.Errorf("Changelog failed, expected error, got nil")
}
+ // Test manifest with remote urls specified and no default URL
+ additions, removals, err = Changelog(httpClient, "1.0.0", "2.0.0", cosInstance, defaultManifestRepo, -1)
+ if additions == nil {
+ t.Errorf("Changelog failed, expected additions, got nil")
+ } else if removals == nil {
+ t.Errorf("Changelog failed, expected removals, got nil")
+ } else if err != nil {
+ t.Errorf("Changelog failed, expected no error, got %v", err)
+ }
+
// Test 1 build number difference with only 1 repo change between them
// Ensure that commits are correctly inserted in proper order
+ // Check that changelog metadata correctly populated
source := "15050.0.0"
target := "15051.0.0"
expectedCommits := []string{
@@ -227,23 +238,29 @@
"9bc12bb411f357188d008864f80dfba43210b9d8",
"bf0dd3757826b9bc9d7082f5f749ff7615d4bcb3",
}
- additions, misses, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo)
+ additions, removals, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo, -1)
if err != nil {
t.Errorf("Changelog failed, expected no error, got %v", err)
- } else if len(misses) != 0 {
- t.Errorf("Changelog failed, expected empty misses list, got %v", misses)
+ } else if len(removals) != 0 {
+ t.Errorf("Changelog failed, expected empty removals list, got %v", removals)
} else if len(additions) != 1 {
t.Errorf("Changelog failed, expected only 1 repo in additions, got %v", additions)
- } else if _, ok := additions["cos/overlays/board-overlays"]; !ok {
- t.Errorf("Changelog failed, expected \"cos/overlays/board-overlays\" in additions, got %v", additions)
- } else if changes, _ := additions["cos/overlays/board-overlays"]; len(changes) != 108 {
+ }
+ boardOverlayLog := additions["cos/overlays/board-overlays"]
+ if boardOverlayLog == nil {
+ t.Errorf("Changelog failed, expected cos/overlays/board-overlays in changelog, got nil")
+ } else if changes := boardOverlayLog.Commits; len(changes) != 108 {
t.Errorf("Changelog failed, expected 108 changes for \"cos/overlays/board-overlays\", got %d", len(changes))
- } else if !commitsMatch(additions["cos/overlays/board-overlays"], expectedCommits) {
+ } else if !commitsMatch(boardOverlayLog.Commits, expectedCommits) {
t.Errorf("Changelog failed, Changelog output does not match expected commits or is not sorted")
+ } else if boardOverlayLog.SourceSHA != "612ca5ef5455534127d008e08c65aa29a2fd97a5" {
+ t.Errorf("Changelog failed, expected SourceSHA \"612ca5ef5455534127d008e08c65aa29a2fd97a5\", got %s", boardOverlayLog.SourceSHA)
+ } else if boardOverlayLog.TargetSHA != "6201c49afe667c8fa7796608a4d7162bb3f7f4f4" {
+ t.Errorf("Changelog failed, expected SourceSHA \"6201c49afe667c8fa7796608a4d7162bb3f7f4f4\", got %s", boardOverlayLog.TargetSHA)
}
// Test build numbers further apart from each other with multiple repo differences
- // Also ensures that misses are correctly populated
+ // Also ensures that removals are correctly populated
source = "15020.0.0"
target = "15056.0.0"
additionRepos := []string{
@@ -266,12 +283,89 @@
"mirrors/cros/chromiumos/repohooks",
"mirrors/cros/chromiumos/overlays/portage-stable",
}
- additions, misses, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo)
+ additions, removals, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo, -1)
if err != nil {
t.Errorf("Changelog failed, expected no error, got %v", err)
- } else if _, ok := misses["third_party/kernel"]; len(misses) != 1 && ok {
- t.Errorf("Changelog failed, expected miss list containing only \"third_party/kernel\", got %v", misses)
- } else if !mappingInLog(additions, additionRepos) {
+ }
+ kernelLog := additions["third_party/kernel"]
+ if len(removals) != 1 || kernelLog == nil {
+ t.Errorf("Changelog failed, expected miss list containing only \"third_party/kernel\", got %v", removals)
+ } else if !repoListInLog(additions, additionRepos) {
t.Errorf("Changelog failed, additions repo output does not match expected repos %v", additionRepos)
}
+
+ // Test changelog returns correct output when given a querySize instead of -1
+ source = "15030.0.0"
+ target = "15050.0.0"
+ querySize := 50
+ additions, removals, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo, querySize)
+ if err != nil {
+ t.Errorf("Changelog failed, expected no error, got %v", err)
+ } else if additions == nil {
+ t.Errorf("Changelog failed, non-empty expected additions, got nil")
+ } else if removals == nil {
+ t.Errorf("Changelog failed, non-empty expected removals, got nil")
+ } else if _, ok := additions["third_party/kernel"]; !ok {
+ t.Errorf("Changelog failed, expected repo: third_party/kernel in additions")
+ }
+ for repoName, repoLog := range additions {
+ if repoLog.Commits == nil || len(repoLog.Commits) == 0 {
+ t.Errorf("Changelog failed, expected non-empty additions commits, got nil or empty commits")
+ }
+ if len(repoLog.Commits) > querySize {
+ t.Errorf("Changelog failed, expected %d commits for repo: %s, got: %d", querySize, repoName, len(repoLog.Commits))
+ } else if repoName == "third_party/kernel" && !repoLog.HasMoreCommits {
+ t.Errorf("Changelog failed, expected HasMoreCommits = True for repo: third_party/kernel, got False")
+ } else if repoLog.HasMoreCommits && len(repoLog.Commits) < querySize {
+ t.Errorf("Changelog failed, expected HasMoreCommits = False for repo: %s with %d commits, got True", repoName, len(repoLog.Commits))
+ }
+ }
+
+ // Test changelog handles manifest with non-matching repositories
+ source = "12871.1177.0"
+ target = "12871.1179.0"
+ additions, removals, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo, querySize)
+ if err != nil {
+ t.Errorf("Changelog failed, expected no error, got %v", err)
+ } else if len(removals) != 0 {
+ t.Errorf("Changelog failed, expected empty removals, got %v", removals)
+ } else if _, ok := additions["cos/cobble"]; !ok {
+ t.Errorf("Changelog failed, expected repo: third_party/kernel in additions")
+ }
+ for repoName, repoLog := range additions {
+ if repoLog.Commits == nil || len(repoLog.Commits) == 0 {
+ t.Errorf("Changelog failed, expected non-empty additions commits, got nil or empty commits")
+ } else if repoName == "cos/cobble" {
+ if repoLog.HasMoreCommits {
+ t.Errorf("Changelog failed, expected hasMoreCommits = false for repo: cos/cobble, got true")
+ } else if repoLog.SourceSHA != "" {
+ t.Errorf("Changelog failed, expected empty SourceSHA for cos/cobble, got %s", repoLog.SourceSHA)
+ } else if repoLog.TargetSHA != "4ab43f1f86b7099b8ad75cf9615ea1fa155bbd7d" {
+ t.Errorf("Changelog failed, expected TargetSHA: \"4ab43f1f86b7099b8ad75cf9615ea1fa155bbd7d\" for cos/cobble, got %s", repoLog.TargetSHA)
+ }
+ }
+ }
+ source = "12871.1179.0"
+ target = "12871.1177.0"
+ additions, removals, err = Changelog(httpClient, source, target, cosInstance, defaultManifestRepo, querySize)
+ if err != nil {
+ t.Errorf("Changelog failed, expected no error, got %v", err)
+ } else if len(additions) != 0 {
+ t.Errorf("Changelog failed, expected empty additions, got %v", additions)
+ } else if _, ok := removals["cos/cobble"]; !ok {
+ t.Errorf("Changelog failed, expected repo: third_party/kernel in additions")
+ }
+ for repoName, repoLog := range removals {
+ if repoLog.Commits == nil || len(repoLog.Commits) == 0 {
+ t.Errorf("Changelog failed, expected non-empty additions commits, got nil or empty commits")
+ } else if repoName == "cos/cobble" {
+ if repoLog.HasMoreCommits {
+ t.Errorf("Changelog failed, expected hasMoreCommits = false for repo: cos/cobble, got true")
+ } else if repoLog.SourceSHA != "" {
+ t.Errorf("Changelog failed, expected empty SourceSHA for cos/cobble, got %s", repoLog.SourceSHA)
+ } else if repoLog.TargetSHA != "4ab43f1f86b7099b8ad75cf9615ea1fa155bbd7d" {
+ t.Errorf("Changelog failed, expected TargetSHA: \"4ab43f1f86b7099b8ad75cf9615ea1fa155bbd7d\" for cos/cobble, got %s", repoLog.TargetSHA)
+ }
+ }
+ }
}
diff --git a/src/pkg/changelog/gitcommit.go b/src/pkg/changelog/gitcommit.go
index e52b2c7..39d49f4 100755
--- a/src/pkg/changelog/gitcommit.go
+++ b/src/pkg/changelog/gitcommit.go
@@ -18,13 +18,22 @@
"errors"
"regexp"
"strings"
- "time"
"go.chromium.org/luci/common/proto/git"
)
-const bugLinePrefix string = "BUG="
-const releaseNoteLinePrefix string = "RELEASE_NOTE="
+const (
+ bugLinePrefix string = "BUG="
+ releaseNoteLinePrefix string = "RELEASE_NOTE="
+)
+
+// RepoLog contains a changelist for a particular repository
+type RepoLog struct {
+ Commits []*Commit
+ SourceSHA string
+ TargetSHA string
+ HasMoreCommits bool
+}
// Commit is a simplified struct of git.Commit
// Useful for interfaces
@@ -104,7 +113,7 @@
func commitTime(commit *git.Commit) string {
if commit.Committer != nil {
- return commit.Committer.Time.AsTime().Format(time.RFC1123)
+ return commit.Committer.Time.AsTime().Format("Mon, 2 Jan 2006")
}
return "None"
}
diff --git a/src/pkg/changelog/gitcommit_test.go b/src/pkg/changelog/gitcommit_test.go
index 268245d..7c623ac 100644
--- a/src/pkg/changelog/gitcommit_test.go
+++ b/src/pkg/changelog/gitcommit_test.go
@@ -29,7 +29,7 @@
parent string = "7645df3136c5b5e43eb1af182b0c67d78ca2d517"
authorName string = "Austin Yuan"
committerName string = "Boston Yuan"
- timeVal string = "Sat, 01 Feb 2020 08:15:00 UTC"
+ timeVal string = "Sat, 1 Feb 2020"
)
var authorTime time.Time
@@ -339,9 +339,9 @@
case !reflect.DeepEqual(commit.Bugs, test.Bugs[i]):
t.Errorf("exptected bugs %#v, got %#v", test.Bugs[i], commit.Bugs)
case commit.ReleaseNote != test.ReleaseNote[i]:
- t.Errorf("expected release note %s, got %s", test.ReleaseNote, commit.ReleaseNote)
+ t.Errorf("expected release note %s, got %s", test.ReleaseNote[i], commit.ReleaseNote)
case commit.CommitTime != test.CommitTime[i]:
- t.Errorf("expected commit time %s, got %s", test.CommitTime, commit.CommitTime)
+ t.Errorf("expected commit time %s, got %s", test.CommitTime[i], commit.CommitTime)
}
}
})
