Add Kernel CI config generation - cos_gpu_config_builder container
Built the container using the supplied Dockerfile and ran the container:
```
docker run --rm config-builder:latest --driver-versions
"450.119.04,470.82.01" --kernel-version 5.15.67-11.m101
```
This produced two files per "config":
gs://cos-gpu-configs/2022-09-20T22:20:19-0032b70a/config.textproto
gs://cos-gpu-configs/2022-09-20T22:20:19-0032b70a/metadata
The config.textproto is the actual build request file:
```
$ gsutil cat gs://cos-gpu-configs/2022-09-20T22:20:19-0032b70a/config.textproto
kernel_src_tarball_gcs: "gs://cos-kernel-artifacts/builds/5.15.67-11.m101/cos-kernel-src-5.15.67-11.m101.tgz"
kernel_headers_tarball_gcs: "gs://cos-kernel-artifacts/builds/5.15.67-11.m101/cos-kernel-headers-5.15.67-11.m101-x86_64.tgz"
nvidia_runfile_address: "https://us.download.nvidia.com/tesla/450.119.04/NVIDIA-Linux-x86_64-450.119.04.run"
toolchain_tarball_gcs: "gs://chromiumos-sdk/2021/06/x86_64-cros-linux-gnu-2021.06.26.094653.tar.xz"
toolchain_env_gcs: "gs://cos-kernel-artifacts/builds/5.15.67-11.m101/toolchain_env.x86_64"
driver_output_gcs_dir: "gs://nvidia-drivers-us-public/nvidia-cos-project/5.15.67-11.m101/"
```
The metadata file:
```
$ gsutil cat gs://cos-gpu-configs/2022-09-20T22:20:19-0032b70a/metadata
{
"driver_version":"450.119.04",
"milestone":"101",
"version":"5.15.67-11.m101",
"version_type:"Kernel"
}
```
BUG=b/238798451
Change-Id: I0a6b0afe24e7e6a6be6eb5be1e5a89770a9be9b6
Reviewed-on: https://cos-review.googlesource.com/c/cos/tools/+/37087
Cloud-Build: GCB Service account <228075978874@cloudbuild.gserviceaccount.com>
Tested-by: Arnav Kansal <rnv@google.com>
Reviewed-by: Robert Kolchmeyer <rkolchmeyer@google.com>
diff --git a/go.mod b/go.mod
index 28a21f1..a4d0bd7 100644
--- a/go.mod
+++ b/go.mod
@@ -3,30 +3,35 @@
go 1.16
require (
+ cloud.google.com/go v0.104.0 // indirect
cloud.google.com/go/compute v1.7.0
cloud.google.com/go/logging v1.4.2
cloud.google.com/go/secretmanager v1.5.0
- cloud.google.com/go/storage v1.22.1
+ cloud.google.com/go/storage v1.26.0
github.com/GoogleCloudPlatform/cloudsql-proxy v1.31.2
github.com/andygrunwald/go-gerrit v0.0.0-20201231163137-46815e48bfe0
github.com/beevik/etree v1.1.0
github.com/go-sql-driver/mysql v1.6.0
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+ github.com/golang/protobuf v1.5.2
github.com/google/go-cmp v0.5.8
github.com/google/subcommands v1.2.0
+ github.com/google/uuid v1.3.0
github.com/gorilla/sessions v1.2.0
github.com/julienschmidt/httprouter v1.3.0 // indirect
github.com/pkg/errors v0.9.1
- github.com/sirupsen/logrus v1.8.1
+ github.com/sirupsen/logrus v1.9.0
github.com/smartystreets/assertions v1.1.1 // indirect
github.com/smartystreets/goconvey v1.6.4 // indirect
+ github.com/stretchr/testify v1.8.0 // indirect
github.com/urfave/cli/v2 v2.2.0
go.chromium.org/luci v0.0.0-20200722211809-bab0c30be68b
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e
- golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c
+ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094
golang.org/x/sys v0.0.0-20220731174439-a90be440212d
- google.golang.org/api v0.90.0
- google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f
- google.golang.org/grpc v1.47.0
- google.golang.org/protobuf v1.28.0
+ google.golang.org/api v0.94.0
+ google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc
+ google.golang.org/grpc v1.48.0
+ google.golang.org/protobuf v1.28.1
)
diff --git a/go.sum b/go.sum
index 09cead3..551895b 100644
--- a/go.sum
+++ b/go.sum
@@ -27,8 +27,10 @@
cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
-cloud.google.com/go v0.102.0 h1:DAq3r8y4mDgyB/ZPJ9v/5VJNqjgJAxTn6ZYLlUywOu8=
cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc=
+cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU=
+cloud.google.com/go v0.104.0 h1:gSmWO7DY1vOm0MVU6DNXM11BWHHsTUmsC5cv1fuW5X8=
+cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
@@ -61,8 +63,10 @@
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-cloud.google.com/go/storage v1.22.1 h1:F6IlQJZrZM++apn9V5/VfS3gbTUYg98PS3EMQAzqtfg=
cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y=
+cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc=
+cloud.google.com/go/storage v1.26.0 h1:lYAGjknyDJirSzfwUlkv4Nsnj7od7foxQNH/fqZqles=
+cloud.google.com/go/storage v1.26.0/go.mod h1:mk/N7YwIKEWyTvXAWQCIeiCTdLoRH6Pd5xmSnolQLTI=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
@@ -137,8 +141,9 @@
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -217,8 +222,6 @@
github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw=
github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
@@ -226,7 +229,6 @@
github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk=
github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
-github.com/googleapis/go-type-adapters v1.0.0 h1:9XdMn+d/G57qq1s8dNc5IesGCXHf6V2HZ2JwRxfA2tA=
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -281,7 +283,6 @@
github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
@@ -326,8 +327,8 @@
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
-github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.1.1 h1:T/YLemO5Yp7KPzS+lVtu+WsHn8yoSwTfItdAd1r3cck=
github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
@@ -337,13 +338,16 @@
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -496,8 +500,9 @@
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
-golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c h1:q3gFqPqH7NVofKo3c3yETAP//pPI+G5mvB7qqj1Y5kY=
golang.org/x/oauth2 v0.0.0-20220722155238-128564f6959c/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
+golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 h1:2o1E+E8TpNLklK9nHiPiK1uzIYrIHt+cQx3ynCwq9V8=
+golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -579,6 +584,7 @@
golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220731174439-a90be440212d h1:Sv5ogFZatcgIMMtBSTTAgMYsicp25MXBubjXNDKwm80=
golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@@ -707,8 +713,10 @@
google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g=
-google.golang.org/api v0.90.0 h1:WMnUWAvihIClUYFNeFA69VTuR3duKS3IalMGDQcLvq8=
google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
+google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
+google.golang.org/api v0.94.0 h1:KtKM9ru3nzQioV1HLlUf1cR7vMYJIpgls5VhAYQXIwA=
+google.golang.org/api v0.94.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -798,8 +806,10 @@
google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f h1:hJ/Y5SqPXbarffmAsApliUlcvMU+wScNGfyop4bZm8o=
google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
+google.golang.org/genproto v0.0.0-20220810155839-1856144b1d9c/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
+google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc h1:Nf+EdcTLHR8qDNN/KfkQL0u0ssxt9OhbaWCl5C0ucEI=
+google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -830,8 +840,9 @@
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8=
google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w=
+google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -846,8 +857,9 @@
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
@@ -858,8 +870,9 @@
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/src/cmd/cos_gpu_config_builder/Dockerfile b/src/cmd/cos_gpu_config_builder/Dockerfile
new file mode 100644
index 0000000..ef2dad4
--- /dev/null
+++ b/src/cmd/cos_gpu_config_builder/Dockerfile
@@ -0,0 +1,10 @@
+FROM golang:1.18 as go-builder
+COPY . /work/
+WORKDIR /work/src/cmd/cos_gpu_config_builder
+ARG TARGETOS
+ARG TARGETARCH
+RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -o cos-gpu-config-builder main.go
+FROM gcr.io/distroless/static-debian11
+LABEL maintainer="cos-containers@google.com"
+COPY --from=go-builder /work/src/cmd/cos_gpu_config_builder/cos-gpu-config-builder /cos-gpu-config-builder
+ENTRYPOINT ["/cos-gpu-config-builder"]
diff --git a/src/cmd/cos_gpu_config_builder/cloudbuild.yaml b/src/cmd/cos_gpu_config_builder/cloudbuild.yaml
new file mode 100644
index 0000000..b817dfb
--- /dev/null
+++ b/src/cmd/cos_gpu_config_builder/cloudbuild.yaml
@@ -0,0 +1,5 @@
+steps:
+- name: 'gcr.io/cloud-builders/docker'
+ args: ['build', '-f', 'src/cmd/cos_gpu_config_builder/Dockerfile', '-t', 'gcr.io/${_OUTPUT_PROJECT}/cos-gpu-config-builder:${TAG_NAME}', '.']
+images:
+- 'gcr.io/${_OUTPUT_PROJECT}/cos-gpu-config-builder:${TAG_NAME}'
diff --git a/src/cmd/cos_gpu_config_builder/main.go b/src/cmd/cos_gpu_config_builder/main.go
new file mode 100644
index 0000000..d54e9b5
--- /dev/null
+++ b/src/cmd/cos_gpu_config_builder/main.go
@@ -0,0 +1,41 @@
+package main
+
+import (
+ "context"
+ "flag"
+ "log"
+ "strings"
+
+ "cloud.google.com/go/storage"
+ "cos.googlesource.com/cos/tools.git/src/pkg/gpuconfig"
+)
+
+var (
+ bucket = flag.String("gcs-bucket", "cos-gpu-configs", "GCS bucket to upload GPU configs to.")
+ kernelVersion = flag.String("kernel-version", "", "Kernel version for COS GPU precompilation build request, example: 5.10.105-23.m97")
+
+ driverVersions = flag.String("driver-versions", "", "Driver version/ (Comma separated if multiple driver versions) for COS GPU precompilation build request, example 450.119.04 / 450.119.04,470.150.03")
+)
+
+func main() {
+ flag.Parse()
+
+ if *kernelVersion == "" || *driverVersions == "" {
+ log.Fatal("empty kernel version: %s or driver version:%s specified", kernelVersion, driverVersions)
+ }
+
+ ctx := context.Background()
+ client, err := storage.NewClient(ctx)
+ if err != nil {
+ log.Fatal("failed to setup client for GCS: %v", err)
+ }
+
+ configs, err := gpuconfig.GenerateKernelCIConfigs(ctx, client, *kernelVersion, strings.Split(*driverVersions, ","))
+ if err != nil {
+ log.Fatal("gpu config generation failed: %v", err)
+ }
+
+ if err := gpuconfig.UploadConfigs(ctx, client, configs, *bucket); err != nil {
+ log.Fatal("uploading gpu config failed: %v", err)
+ }
+}
diff --git a/src/pkg/gcs/gcs_client.go b/src/pkg/gcs/gcs_client.go
index df833a0..5ddf534 100644
--- a/src/pkg/gcs/gcs_client.go
+++ b/src/pkg/gcs/gcs_client.go
@@ -51,22 +51,31 @@
}
// UploadGCSObject uploads an object at inputPath to destination URL
-func UploadGCSObject(ctx context.Context,
- gcsClient *storage.Client, inputPath, destinationURL string) error {
-
- gcsBucket, name, err := getGCSVariables(destinationURL)
- if err != nil {
- return fmt.Errorf("error parsing destination URL: %v", err)
- }
+func UploadGCSObject(ctx context.Context, gcsClient *storage.Client, inputPath, destinationURL string) error {
fileReader, err := os.Open(inputPath)
if err != nil {
return err
}
+ return uploadGCSObject(ctx, gcsClient, fileReader, destinationURL)
+}
+// UploadGCSObjectString uploads an input string as a file to destination URL
+func UploadGCSObjectString(ctx context.Context, gcsClient *storage.Client, inputStr, destinationURL string) error {
+ reader := strings.NewReader(inputStr)
+ return uploadGCSObject(ctx, gcsClient, reader, destinationURL)
+}
+
+func uploadGCSObject(ctx context.Context,
+ gcsClient *storage.Client, reader io.Reader, destinationURL string) error {
+ gcsBucket, name, err := getGCSVariables(destinationURL)
+ if err != nil {
+ return fmt.Errorf("error parsing destination URL: %v", err)
+ }
w := gcsClient.Bucket(gcsBucket).Object(name).NewWriter(ctx)
- defer w.Close()
-
- if _, err := io.Copy(w, fileReader); err != nil {
+ if _, err := io.Copy(w, reader); err != nil {
+ return err
+ }
+ if err := w.Close(); err != nil {
return err
}
return nil
diff --git a/src/pkg/gpuconfig/generate_configs.go b/src/pkg/gpuconfig/generate_configs.go
index 23d19a8..d7fd5e0 100644
--- a/src/pkg/gpuconfig/generate_configs.go
+++ b/src/pkg/gpuconfig/generate_configs.go
@@ -6,3 +6,83 @@
package gpuconfig
//go:generate protoc --go_out=:./pb -I. proto/config.proto
+
+import (
+ "context"
+ "fmt"
+ "io/ioutil"
+ "strings"
+
+ "cloud.google.com/go/storage"
+ "cos.googlesource.com/cos/tools.git/src/pkg/gpuconfig/pb"
+)
+
+const (
+ kernelGCSPrefix string = "gs://cos-kernel-artifacts/builds"
+ kernelSrcTarballPathTemplate string = "%s/%[2]s/cos-kernel-src-%[2]s.tgz"
+ kernelHeadersTarballPathTemplate string = "%s/%[2]s/cos-kernel-headers-%[2]s-x86_64.tgz"
+ toolchainTarballPathTemplate string = "builds/%s/toolchain_url.x86_64"
+ toolchainEnvPathTemplate string = "%s/%s/toolchain_env.x86_64"
+ driverOutputGcsDirTemplate string = "gs://nvidia-drivers-us-public/nvidia-cos-project/%s/"
+ nvidiaRunfileAddressTemplate string = "https://us.download.nvidia.com/tesla/%[1]s/NVIDIA-Linux-x86_64-%[1]s.run"
+ timeFormatTemplate string = "2006-01-02-15:04:05"
+)
+
+type GPUPrecompilationConfig struct {
+ ProtoConfig *pb.COSGPUBuildRequest `json:"-"`
+ DriverVersion string `json:"driver_version"`
+ Milestone string `json:"milestone"`
+ Version string `json:"version"`
+ VersionType string `json:"version_type"`
+}
+
+// Generates and GPU precompilation build configs(and metadata) for a given
+// tuple of kernelVersion and driver versions
+func GenerateKernelCIConfigs(ctx context.Context, client *storage.Client, kernelVersion string, driverVersions []string) ([]GPUPrecompilationConfig, error) {
+ configs := []GPUPrecompilationConfig{}
+ for _, driverVersion := range driverVersions {
+ config, err := constructKernelCIConfig(ctx, client, kernelVersion, driverVersion)
+ if err != nil {
+ return nil, err
+ }
+ milestone := strings.Split(kernelVersion, "m")[1]
+ configs = append(configs, GPUPrecompilationConfig{config, driverVersion, milestone, kernelVersion, "Kernel"})
+ }
+ return configs, nil
+}
+
+func constructKernelCIConfig(ctx context.Context, client *storage.Client, kernelVersion, driverVersion string) (*pb.COSGPUBuildRequest, error) {
+ config := pb.COSGPUBuildRequest{
+ KernelSrcTarballGcs: stringPtr(fmt.Sprintf(kernelSrcTarballPathTemplate, kernelGCSPrefix, kernelVersion)),
+ KernelHeadersTarballGcs: stringPtr(fmt.Sprintf(kernelHeadersTarballPathTemplate, kernelGCSPrefix, kernelVersion)),
+ NvidiaRunfileAddress: stringPtr(fmt.Sprintf(nvidiaRunfileAddressTemplate, driverVersion)),
+ ToolchainEnvGcs: stringPtr(fmt.Sprintf(toolchainEnvPathTemplate, kernelGCSPrefix, kernelVersion)),
+ DriverOutputGcsDir: stringPtr(fmt.Sprintf(driverOutputGcsDirTemplate, kernelVersion)),
+ }
+
+ toolchainTarballPath, err := fetchToolchainTarballPath(ctx, client, kernelVersion)
+ if err != nil {
+ return nil, err
+ }
+ config.ToolchainTarballGcs = &toolchainTarballPath
+
+ return &config, nil
+}
+
+func fetchToolchainTarballPath(ctx context.Context, client *storage.Client, kernelVersion string) (string, error) {
+ toolchainTarballPathURL := fmt.Sprintf(toolchainTarballPathTemplate, kernelVersion)
+ reader, err := client.Bucket("cos-kernel-artifacts").Object(toolchainTarballPathURL).NewReader(ctx)
+ if err != nil {
+ return "", fmt.Errorf("Could not fetch the toolchain tarball path: %w", err)
+ }
+ var toolchainTarballPath []byte
+ if toolchainTarballPath, err = ioutil.ReadAll(reader); err != nil {
+ return "", fmt.Errorf("Could not read file contents of toolchain tarball path: %w", err)
+ }
+ return string(toolchainTarballPath), nil
+}
+
+// stringPtr returns a pointer to a string.
+func stringPtr(s string) *string {
+ return &s
+}
diff --git a/src/pkg/gpuconfig/generate_configs_test.go b/src/pkg/gpuconfig/generate_configs_test.go
new file mode 100644
index 0000000..c9c8243
--- /dev/null
+++ b/src/pkg/gpuconfig/generate_configs_test.go
@@ -0,0 +1,74 @@
+package gpuconfig
+
+import (
+ "context"
+ "testing"
+
+ "cos.googlesource.com/cos/tools.git/src/pkg/fakes"
+ "cos.googlesource.com/cos/tools.git/src/pkg/gpuconfig/pb"
+ "github.com/google/go-cmp/cmp"
+ "google.golang.org/protobuf/testing/protocmp"
+)
+
+const toolchainTarballPath = "gs://chromiumos-sdk/2021/06/x86_64-cros-linux-gnu-2021.06.26.094653.tar.xz"
+
+var testGCSObjects = map[string][]byte{
+ "/cos-kernel-artifacts/builds/5.15.55-34.m101/toolchain_url.x86_64": []byte(toolchainTarballPath),
+}
+
+func TestGenerateKernelCIConfigs(t *testing.T) {
+ gcs := fakes.GCSForTest(t)
+ defer gcs.Close()
+ gcs.Objects = testGCSObjects
+ client := gcs.Client
+ for _, tc := range []struct {
+ kernelVersion string
+ driverVersions []string
+ expected []GPUPrecompilationConfig
+ }{
+ {
+ "5.15.55-34.m101",
+ []string{"470.82.01"},
+ []GPUPrecompilationConfig{
+ GPUPrecompilationConfig{
+ ProtoConfig: &pb.COSGPUBuildRequest{
+ KernelSrcTarballGcs: stringPtr("gs://cos-kernel-artifacts/builds/5.15.55-34.m101/cos-kernel-src-5.15.55-34.m101.tgz"),
+ KernelHeadersTarballGcs: stringPtr("gs://cos-kernel-artifacts/builds/5.15.55-34.m101/cos-kernel-headers-5.15.55-34.m101-x86_64.tgz"),
+ NvidiaRunfileAddress: stringPtr("https://us.download.nvidia.com/tesla/470.82.01/NVIDIA-Linux-x86_64-470.82.01.run"),
+ ToolchainTarballGcs: stringPtr("gs://chromiumos-sdk/2021/06/x86_64-cros-linux-gnu-2021.06.26.094653.tar.xz"),
+ ToolchainEnvGcs: stringPtr("gs://cos-kernel-artifacts/builds/5.15.55-34.m101/toolchain_env.x86_64"),
+ DriverOutputGcsDir: stringPtr("gs://nvidia-drivers-us-public/nvidia-cos-project/5.15.55-34.m101/"),
+ },
+ DriverVersion: "470.82.01",
+ Milestone: "101",
+ Version: "5.15.55-34.m101",
+ VersionType: "Kernel",
+ },
+ },
+ },
+ } {
+ ctx := context.Background()
+ got, err := GenerateKernelCIConfigs(ctx, client, tc.kernelVersion, tc.driverVersions)
+ if err != nil {
+ t.Fatalf("GenerateKernelCIConfig() failed: %s", err)
+ }
+ if diff := cmp.Diff(got, tc.expected, protocmp.Transform()); diff != "" {
+ t.Errorf("GenerateKernelCIConfig() returned unexpected difference (-want +got):\n%s", diff)
+ }
+ }
+}
+
+func TestFetchToolchainTarballPath(t *testing.T) {
+ gcs := fakes.GCSForTest(t)
+ defer gcs.Close()
+ gcs.Objects = testGCSObjects
+ client := gcs.Client
+ kernelVersion := "5.15.55-34.m101"
+ got, err := fetchToolchainTarballPath(context.Background(), client, kernelVersion)
+ if err != nil {
+ t.Fatalf("fetchToolchainTarballPath() failed: %s", err)
+ }
+ if got != toolchainTarballPath {
+ t.Errorf("fetchToolchainTarballPath() = %+v, want %+v", got, toolchainTarballPath)
+ }
+}
diff --git a/src/pkg/gpuconfig/upload_configs.go b/src/pkg/gpuconfig/upload_configs.go
new file mode 100644
index 0000000..ab7145f
--- /dev/null
+++ b/src/pkg/gpuconfig/upload_configs.go
@@ -0,0 +1,36 @@
+package gpuconfig
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log"
+ "strings"
+ "time"
+
+ "cloud.google.com/go/storage"
+ "cos.googlesource.com/cos/tools.git/src/pkg/gcs"
+ "github.com/golang/protobuf/proto"
+ "github.com/google/uuid"
+)
+
+func destDir(gcsBucket string) string {
+ timestamp := strings.TrimSuffix(time.Now().Format(time.RFC3339), "Z")
+ uid := uuid.NewString()[:8]
+ return fmt.Sprintf("gs://%s/%s", gcsBucket, timestamp+"-"+uid)
+}
+
+func UploadConfigs(ctx context.Context, client *storage.Client, configs []GPUPrecompilationConfig, gcsBucket string) error {
+ for _, config := range configs {
+ log.Printf("uploading gpu precompilation config for: %s, driver version %s\n", config.Version, config.DriverVersion)
+ destDir := destDir(gcsBucket)
+ if err := gcs.UploadGCSObjectString(ctx, client, proto.MarshalTextString(config.ProtoConfig), fmt.Sprintf("%s/%s", destDir, "config.textproto")); err != nil {
+ return err
+ }
+ metadata, _ := json.MarshalIndent(config, "", " ")
+ if err := gcs.UploadGCSObjectString(ctx, client, string(metadata), fmt.Sprintf("%s/%s", destDir, "metadata")); err != nil {
+ return err
+ }
+ }
+ return nil
+}
