Modify utils package to include parsing APIs
This CL defines logic for running commands and parsing the
output of such commands. It also defines a command interface
that can be implemented to execute shell commands.
Change-Id: I8bfbe6b115bc27399231e049ae32090f024ff508
Reviewed-on: https://cos-review.googlesource.com/c/cos/tools/+/18810
Cloud-Build: GCB Service account <228075978874@cloudbuild.gserviceaccount.com>
Reviewed-by: Dexter Rivera <riverade@google.com>
Reviewed-by: Vaibhav Rustagi <vaibhavrustagi@google.com>
Tested-by: Dexter Rivera <riverade@google.com>
diff --git a/src/pkg/nodeprofiler/profiler/commands.go b/src/pkg/nodeprofiler/profiler/commands.go
new file mode 100644
index 0000000..a41d779
--- /dev/null
+++ b/src/pkg/nodeprofiler/profiler/commands.go
@@ -0,0 +1,90 @@
+package profiler
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+
+ "cos.googlesource.com/cos/tools.git/src/pkg/nodeprofiler/utils"
+)
+
+// Command interface defines functions that can be implemented by
+// structs to execute shell commands.
+type Command interface {
+ Name() string
+ Run(opts Options) (map[string][]string, error)
+}
+
+// Options stores the options that can be passed to a command and
+// to parsing functions.
+type Options struct {
+ // Delay specifies times between updates in seconds.
+ Delay int
+ // Count specifies number of updates.
+ Count int
+ // Titles specifies the titles to get values for.
+ Titles []string
+}
+
+// vmstat represents a vmstat command.
+type vmstat struct {
+ name string
+}
+
+// lscpu represents an lscpu command.
+type lscpu struct {
+ name string
+}
+
+// Name returns the name for vmstat command.
+func (v *vmstat) Name() string {
+ return v.name
+}
+
+// Run executes the vmstat command, parses the output and returns it as
+// a map of titles to their values.
+func (v *vmstat) Run(opts Options) (map[string][]string, error) {
+ // delay and count not set
+ if opts.Delay == 0 {
+ opts.Delay = 1
+ }
+ if opts.Count == 0 {
+ opts.Count = 5
+ }
+ interval := strconv.Itoa(opts.Delay)
+ count := strconv.Itoa(opts.Count)
+ out, err := utils.RunCommand(v.Name(), "-n", interval, count)
+ if err != nil {
+ return nil, fmt.Errorf("failed to run vmstat command: %v", err)
+ }
+
+ s := string(out)
+ lines := strings.Split(strings.Trim(s, "\n"), "\n")
+ // ignore the first row in vmstat's output
+ lines = lines[1:]
+ titles := opts.Titles
+ // parse output by columns
+ output, err := utils.ParseColumns(lines, titles...)
+ return output, err
+
+}
+
+// Name returns the name for the lscpu command.
+func (l *lscpu) Name() string {
+ return l.name
+}
+
+// Run executes the lscpu command, parses the output and returns a
+// a map of title(s) to their values.
+func (l *lscpu) Run(opts Options) (map[string][]string, error) {
+ out, err := utils.RunCommand(l.Name())
+ if err != nil {
+ return nil, fmt.Errorf("failed to run vmstat command: %v", err)
+ }
+ s := string(out)
+ lines := strings.Split(strings.Trim(s, "\n"), "\n")
+ titles := opts.Titles
+ // parse output by rows
+ output, err := utils.ParseRows(lines, titles...)
+ return output, err
+}
diff --git a/src/pkg/nodeprofiler/profiler/commands_test.go b/src/pkg/nodeprofiler/profiler/commands_test.go
new file mode 100644
index 0000000..fcb8b80
--- /dev/null
+++ b/src/pkg/nodeprofiler/profiler/commands_test.go
@@ -0,0 +1,83 @@
+package profiler
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func TestRun(t *testing.T) {
+ tests := []struct {
+ name string
+ fakeCmd Command
+ opts Options
+ want map[string][]string
+ wantErr bool
+ }{
+ {
+ name: "vmstat",
+ fakeCmd: &vmstat{"testdata/vmstat.sh"},
+ opts: Options{
+ Delay: 1,
+ Count: 3,
+ Titles: []string{"us", "st", "sy"},
+ },
+ want: map[string][]string{
+ "us": {"1", "2", "7"},
+ "sy": {"0", "1", "3"},
+ "st": {"0", "0", "0"},
+ },
+ },
+ {
+ name: "lscpu",
+ fakeCmd: &lscpu{"testdata/lscpu.sh"},
+ opts: Options{
+ Titles: []string{"CPU(s)"},
+ },
+ want: map[string][]string{
+ "CPU(s)": {"8"},
+ },
+ },
+ {
+ name: "no titles",
+ fakeCmd: &vmstat{"testdata/vmstat.sh"},
+ opts: Options{
+ Delay: 1,
+ Count: 2,
+ },
+ want: map[string][]string{
+ "r": {"3", "1"},
+ "us": {"1", "2"},
+ "sy": {"0", "1"},
+ "id": {"96", "98"},
+ "wa": {"3", "0"},
+ "st": {"0", "0"},
+ },
+ },
+ {
+ name: "spaced rows",
+ fakeCmd: &vmstat{"testdata/vmstat.sh"},
+ opts: Options{
+ Delay: 1,
+ Count: 4,
+ Titles: []string{"us", "st", "sy"},
+ },
+ want: map[string][]string{
+ "us": {"7", "3", "1", "1"},
+ "sy": {"2", "2", "2", "0"},
+ "st": {"0", "0", "0", "0"},
+ },
+ },
+ }
+
+ for _, test := range tests {
+ got, err := test.fakeCmd.Run(test.opts)
+
+ if gotErr := err != nil; gotErr != test.wantErr {
+ t.Fatalf("Run(%v) err %q, wantErr %v", test.opts, err, test.wantErr)
+ }
+ if diff := cmp.Diff(test.want, got); diff != "" {
+ t.Errorf("Ran Run(%v), but got mismatch between got and want (-got, +want): \n diff %s", test.opts, diff)
+ }
+ }
+}
diff --git a/src/pkg/nodeprofiler/profiler/testdata/lscpu.sh b/src/pkg/nodeprofiler/profiler/testdata/lscpu.sh
new file mode 100755
index 0000000..ed68e82
--- /dev/null
+++ b/src/pkg/nodeprofiler/profiler/testdata/lscpu.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+main() {
+ cat <<EOF
+ Architecture: x86_64
+ CPU op-mode(s): 32-bit, 64-bit
+ Byte Order: Little Endian
+ Address sizes: 39 bits physical, 48 bits virtual
+ CPU(s): 8
+ On-line CPU(s) list: 0-7
+ Thread(s) per core: 1
+ Core(s) per socket: 8
+ Socket(s): 1
+ Vendor ID: GenuineIntel
+ CPU family: 6
+ Model: 142
+ Model name: 06/8e
+ Stepping: 12
+ CPU MHz: 2303.980
+ BogoMIPS: 4607.96
+ Virtualization: VT-x
+ Hypervisor vendor: KVM
+ Virtualization type: full
+ L1i cache: 32K
+ L2 cache: 256K
+ L3 cache: 8192K
+ Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid tsc_known_freq pni pclmulqdq vmx ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves arat umip md_clear arch_capabilities
+EOF
+}
+
+main "$#"
\ No newline at end of file
diff --git a/src/pkg/nodeprofiler/profiler/testdata/vmstat.sh b/src/pkg/nodeprofiler/profiler/testdata/vmstat.sh
new file mode 100755
index 0000000..125066c
--- /dev/null
+++ b/src/pkg/nodeprofiler/profiler/testdata/vmstat.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+main () {
+ if [[ "$#" -ne 3 ]]; then
+ echo "command not called with 3 arguments" >&2 return 1
+ fi
+ if [[ $3 -lt 0 ]]; then
+ echo "$3 is not a valid argument" >&2 return 1
+ fi
+ case "$3" in
+ "3")
+ cat << EOF
+ procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
+ r b swpd free buff cache si so bi bo in cs us sy id wa st
+ 5 0 0 14827096 0 25608 0 0 2 5 57 2 1 0 99 0 0
+ 2 0 0 14827096 0 25608 0 0 0 0 1131 1594 2 1 97 0 0
+ 2 0 0 14827096 0 25608 0 0 0 0 5283 8037 7 3 90 0 0
+EOF
+ ;;
+ "0")
+ cat <<EOF
+ procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
+ r b swpd free buff cache si so bi bo in cs us sy id wa st
+ 9 0 0 14828152 0 25740 0 0 5 5 69 98 1 0 90 9 0
+EOF
+ ;;
+ "+5")
+ cat <<EOF
+ procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
+ r b swpd free buff cache si so bi bo in cs us sy id wa st
+ 3 0 0 14828112 0 25740 0 0 3 6 85 121 1 0 93 6 0
+ 1 0 0 14828112 0 25740 0 0 0 0 854 1098 1 1 98 0 0
+ 1 0 0 14828112 0 25740 0 0 0 0 1012 1399 2 1 98 0 0
+ 1 0 0 14828112 0 25740 0 0 0 0 2991 4478 5 2 92 0 0
+ 2 0 0 14828112 0 25740 0 0 0 0 7698 8623 18 6 75 0 1
+EOF
+ ;;
+ "1")
+ cat <<EOF
+ procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
+ r b swpd free buff cache si so bi bo in cs us sy id wa st
+ 2 0 0 14827724 0 25608 0 0 1 6 10 37 1 0 96 2 0
+EOF
+ ;;
+ "4")
+ cat <<EOF
+ procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu-----
+ r b swpd free buff cache si so bi bo in cs us sy id wa st
+
+
+ 1 0 0 14827712 0 25608 0 0 0 0 2561 3731 7 2 91 0 0
+ 0 0 0 14827712 0 25608 0 0 0 0 1885 2684 3 2 95 0 0
+
+ 0 0 0 14827712 0 25608 0 0 0 0 827 894 1 2 98 0 0
+
+ 5 0 0 14827780 0 25740 0 0 0 5 3 7 1 0 96 3 0
+EOF
+ ;;
+ "2")
+ cat <<EOF
+ procs--sys--cpu--
+ r us sy id wa st
+ 3 1 0 96 3 0
+ 1 2 1 98 0 0
+EOF
+ ;;
+ esac
+}
+
+main "$@"
+
+
diff --git a/src/pkg/nodeprofiler/utils/parsing.go b/src/pkg/nodeprofiler/utils/parsing.go
new file mode 100644
index 0000000..540f46e
--- /dev/null
+++ b/src/pkg/nodeprofiler/utils/parsing.go
@@ -0,0 +1,108 @@
+package utils
+
+import (
+ "fmt"
+ "strings"
+)
+
+// ParsedOutput is a data structure that holds the parsed output
+// of certain shell commands whose output takes the form of a
+// table.
+type ParsedOutput map[string][]string
+
+// ParseColumns parses command outputs which are in a column table.
+// It takes in an optional titles argument which specifies the
+// columns to parse. If this argument is missing, then all columns are parsed.
+//
+// Eg, ParseColumns(["total used free shared",
+// "14868916 12660 14830916 0"],
+// ["total", "used"]) = map[string][]string {
+// "total": ["14868916"]
+// "used": ["12660"]}
+func ParseColumns(rows []string, titles ...string) (map[string][]string, error) {
+ // parsedOutput is a map that stores col titles as key and entries as values.
+ parsedOutput := ParsedOutput{}
+ // maps column title to its index eg columns["r"] = 0 wth vmstat.
+ columns := make(map[string]int)
+ for i, row := range rows {
+ // break the row into slice
+ tokens := strings.Fields(row)
+ if len(tokens) == 0 {
+ continue
+ }
+ // find index of column titles
+ if i == 0 {
+ // if no titles were specified, use all of them
+ if len(titles) == 0 {
+ titles = tokens
+ }
+ // map header name to its index
+ for index, str := range tokens {
+ columns[str] = index
+ }
+ continue
+ }
+ // loop over titles, get column number of the title using map,
+ // use that to get the actual values, append to list.
+ for _, title := range titles {
+ // for example columns["us"] = 12
+ index, ok := columns[title]
+ if !ok {
+ return nil, fmt.Errorf("unknown Column title %s", title)
+ }
+ // for example if vmstat was run, tokens[0] will give the
+ // value of a running process for some update.
+ value := tokens[index]
+ parsedOutput[title] = append(parsedOutput[title], value)
+ }
+ }
+ return parsedOutput, nil
+}
+
+// ParseRows parses command outputs which are in a row table.
+// It takes in an optional titles argument which specifies which rows
+// to parse. If missing, all rows are parsed.
+//
+// Eg, ParseRows(["avg-cpu: %user %nice %system %iowait %steal %idle"],
+// [avg-cpu]) = map[string][]string {
+// avg-cpu: [%user, %nice, %system, %iowait, %steal, %idle]}
+func ParseRows(lines []string, titles ...string) (map[string][]string, error) {
+ // parsedOutput stores titles passed to function as key and entries as values.
+ parsedOutput := ParsedOutput{}
+ // rows stores each title in rows as key and the rest of the row as value.
+ rows := make(map[string][]string)
+ // loop over lines and map each line title to value(s)
+ for _, line := range lines {
+ // split by ':' for titles that are multi worded
+ tokens := strings.Split(line, ":")
+ // tokens is always at least of length 1 since an empty string when
+ // split, will be a slice of length 1.
+ if len(tokens) == 1 {
+ continue
+ }
+ header := strings.Trim(tokens[0], "\\s*")
+ // everything to the right of title is one string since
+ // row was split by the character ':'.
+ value := tokens[1]
+ // now split value according to white spaces
+ tokens = strings.Fields(value)
+ rows[header] = tokens
+ }
+ // if empty titles slice was passed, use all the row titles.
+ if len(titles) == 0 {
+ for key := range rows {
+ titles = append(titles, key)
+ }
+ }
+ // loop over titles passed to function and get their values from the map.
+ for _, title := range titles {
+ var values []string
+ var ok bool
+ // check if any additional titles were passed in.
+ if values, ok = rows[title]; !ok {
+ return nil, fmt.Errorf("could not find the row title %s", title)
+ }
+ parsedOutput[title] = values
+ }
+ return parsedOutput, nil
+}
diff --git a/src/pkg/nodeprofiler/utils/parsing_test.go b/src/pkg/nodeprofiler/utils/parsing_test.go
new file mode 100644
index 0000000..73ff963
--- /dev/null
+++ b/src/pkg/nodeprofiler/utils/parsing_test.go
@@ -0,0 +1,184 @@
+package utils
+
+import (
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func TestParseColumns(t *testing.T) {
+ tests := []struct {
+ name string
+ rows []string
+ titles []string
+ want map[string][]string
+ wantErr bool
+ }{
+
+ {
+ name: "basic",
+ rows: []string{
+ "r b swpd free buff cache si so bi bo in cs us sy id wa st",
+ "5 0 0 14827096 0 25608 0 0 2 5 57 2 1 0 99 0 0",
+ "2 0 0 14827096 0 25608 0 0 0 0 1131 1594 2 1 97 0 1",
+ "2 0 0 14827096 0 25608 0 0 0 0 5283 8037 7 3 90 0 0",
+ },
+ titles: []string{"us", "sy", "st"},
+ want: map[string][]string{
+ "us": {"1", "2", "7"},
+ "sy": {"0", "1", "3"},
+ "st": {"0", "1", "0"},
+ },
+ },
+ {
+ name: "spaced rows",
+ rows: []string{
+ "r b swpd free buff cache si so bi bo in cs us sy id wa st",
+ " ",
+ "2 0 0 14827096 0 25608 0 0 0 0 1131 1594 2 1 97 0 0",
+ " ",
+ "5 0 0 14827780 0 25740 0 0 0 5 3 7 1 0 96 3 0",
+ " ",
+ "1 0 0 14827724 0 25608 0 0 1 6 10 37 1 0 96 2 0",
+ },
+ titles: []string{"r"},
+ want: map[string][]string{
+ "r": {"2", "5", "1"},
+ },
+ },
+ {
+ name: "unknown titles",
+ rows: []string{
+ "r b swpd free buff cache si so bi bo in cs us sy id wa st",
+ "5 0 0 14827096 0 25608 0 0 2 5 57 2 1 0 99 0 0",
+ },
+ titles: []string{"us", "sy", "steal"},
+ wantErr: true,
+ },
+ {
+ name: "empty slice",
+ rows: []string{},
+ want: map[string][]string{},
+ },
+ {
+ name: "empty titles",
+ rows: []string{
+ "r b swpd free buff",
+ "5 0 0 14827096 0",
+ },
+ want: map[string][]string{
+ "r": {"5"},
+ "b": {"0"},
+ "swpd": {"0"},
+ "free": {"14827096"},
+ "buff": {"0"},
+ },
+ },
+ }
+
+ for _, test := range tests {
+ got, err := ParseColumns(test.rows, test.titles...)
+ if gotErr := err != nil; gotErr != test.wantErr {
+ t.Fatalf("ParseColumns(%v, %v) err %q, wantErr: %v", test.rows, test.titles, err, test.wantErr)
+ }
+ if diff := cmp.Diff(test.want, got); diff != "" {
+ t.Errorf("Ran ParseColumns(%v, %v), but got mismatch between got and want (-got, +want): \n diff %s", test.rows, test.titles, diff)
+ }
+ }
+}
+
+func TestParseRows(t *testing.T) {
+ tests := []struct {
+ name string
+ rows []string
+ titles []string
+ want map[string][]string
+ wantErr bool
+ }{
+ {
+ name: "lscpu's output",
+ rows: []string{
+ "Architecture: x86_64",
+ "CPU op-mode(s): 32-bit, 64-bit",
+ "Byte Order: Little Endian",
+ "Address sizes: 39 bits physical, 48 bits virtual",
+ "CPU(s): 8",
+ "On-line CPU(s) list: 0-7",
+ },
+
+ titles: []string{"CPU(s)"},
+ want: map[string][]string{
+ "CPU(s)": {"8"},
+ },
+ },
+ {
+ name: "spaced rows",
+ rows: []string{
+ "Architecture: x86_64",
+ " ",
+ "CPU op-mode(s): 32-bit, 64-bit",
+ " ",
+ "CPU(s): 8",
+ },
+ titles: []string{"CPU(s)"},
+ want: map[string][]string{
+ "CPU(s)": {"8"},
+ },
+ },
+ {
+ name: "free's output",
+ rows: []string{
+ "Mem: 14518 13 14480 0 25 14505",
+ "Swap: 0 0 0 ",
+ },
+ titles: []string{"Mem", "Swap"},
+ want: map[string][]string{
+ "Mem": {"14518", "13", "14480", "0", "25", "14505"},
+ "Swap": {"0", "0", "0"},
+ },
+ },
+ {
+ name: "unknown titles",
+ rows: []string{
+ "Device: tps kB_read/s kB_wrtn/s kB_read kB_wrtn",
+ "vdb: 0.60 1.54 21.54 836516 11703972",
+ "vda: 0.00 0.07 0.00 37901 0",
+ },
+ titles: []string{"sda"},
+ wantErr: true,
+ },
+ {
+ name: "empty slice",
+ rows: []string{},
+ want: map[string][]string{},
+ },
+ {
+ name: "empty titles",
+ rows: []string{
+ "processor: 0",
+ "vendor_id: GenuineIntel",
+ "cpu family: 6",
+ "model: 142",
+ "model name: 06/8e",
+ },
+ titles: []string{},
+ want: map[string][]string{
+ "processor": {"0"},
+ "vendor_id": {"GenuineIntel"},
+ "cpu family": {"6"},
+ "model": {"142"},
+ "model name": {"06/8e"},
+ },
+ },
+ }
+
+ for _, test := range tests {
+ got, err := ParseRows(test.rows, test.titles...)
+ if gotErr := err != nil; gotErr != test.wantErr {
+ t.Fatalf("ParseRows(%v, %v) = %q, wantErr: %v", test.rows, test.titles, err, test.wantErr)
+ }
+ if diff := cmp.Diff(test.want, got); diff != "" {
+ t.Errorf("Ran ParseRows(%v, %v), but got mismatch between got and want (-got, +want): \n diff %s", test.rows, test.titles, diff)
+ }
+ }
+}
diff --git a/src/pkg/nodeprofiler/utils/utils.go b/src/pkg/nodeprofiler/utils/utils.go
index 8b07b71..a41873d 100644
--- a/src/pkg/nodeprofiler/utils/utils.go
+++ b/src/pkg/nodeprofiler/utils/utils.go
@@ -1,74 +1,18 @@
-// Package utils defines common structs used by the COS Node Profiler Agent
+// Package utils defines parsing and run command functions
+// that can be used outside nodeprofiler.
package utils
-import "time"
+import (
+ "fmt"
+ "os/exec"
+)
-// USEMetrics contain the USE metrics (utilization, saturation, errors)
-// for a particular component of the system.
-type USEMetrics struct {
- // Timestamp refers to the point in time when the USE metrics for this
- // component was collected.
- Timestamp time.Time
- // Interval refers to the time interval for which the USE metrics for this
- // component was collected.
- Interval time.Duration
- // Utilization is the percent over a time interval for which the resource
- // was busy servicing work.
- Utilization float64
- // Saturation is the degree to which the resource has extra work which it
- // can’t service. The value for Saturation has different meanings
- // depending on the component being analyzed. But for simplicity sake
- // Saturation here is just a bool which tells us whether this specific
- // component is saturated or not.
- Saturation bool
- // Errors is the number of errors seen in the component over a given
- // time interval.
- Errors int64
-}
-
-// USEReport contains the USE Report from a single run of the node profiler.
-// The USE Report contains helpful information to help diagnose performance
-// issues seen by customers on their k8s clusters.
-type USEReport struct {
- // Components contains the USE Metrics for each component of the system.
- // Such components include CPU, memory, network, storage, etc.
- Components []USEMetrics
- // Analysis provides insights into the USE metrics collected, including
- // a guess as to which component may be causing performance issues.
- Analysis string
-}
-
-// ProfilerReport contains debugging information provided by the profiler
-// tool. Currently, it will only provide USEMetrics (Utilization,
-// Saturation, Errors), kernel trace outputs, and the outputs of
-// arbitrary shell commands provided by the user.
-// In future, we can add following different types of dynamic reports:
-//
-// type PerfReport - Captures perf command output
-// type STraceReport - Captures strace output
-// type BPFTraceReport - Allows users to add eBPF hooks and capture its
-// output
-type ProfilerReport struct {
- // Static reports
- // USEMetrics provides Utilization, Saturation and Errors for different
- // components on the system
- USEInfo USEReport
- // RawCommands captures the output of arbitrary shell commands provided
- // by the user. Example usage: count # of systemd units; count # of
- // cgroups
- RawCommands map[string][]byte
- // Dynamic tracing reports
- // KernelTraces captures the output of the ftrace command. The key is the
- // kernel trace point and the value is the output
- KernelTraces map[string][]byte
-}
-
-// ProfilerConfig tells the profiler which dynamic reports it should
-// generate and capture.
-type ProfilerConfig struct {
- // KernelTracePoints are the trace points we should insert and capture.
- KernelTracePoints []string
- // RawCommands are the shell commands that we should run and capture
- // output.
- RawCommands []string
+// RunCommand is a wrapper function for exec.Command that will run the command
+// specified return its output and/or error.
+func RunCommand(cmd string, args ...string) ([]byte, error) {
+ out, err := exec.Command(cmd, args...).CombinedOutput()
+ if err != nil {
+ return nil, fmt.Errorf("failed to run %s, %v: %v", cmd, args, err)
+ }
+ return out, nil
}
