| #!/usr/bin/env bash |
| |
| # Copyright 2017 The Kubernetes Authors. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| # A library of helper functions and constant for the local config. |
| |
| # Use the config file specified in $KUBE_CONFIG_FILE, or default to |
| # config-default.sh. |
| readonly GCE_MAX_LOCAL_SSD=8 |
| |
| KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. |
| source "${KUBE_ROOT}/cluster/gce/${KUBE_CONFIG_FILE-"config-default.sh"}" |
| source "${KUBE_ROOT}/cluster/common.sh" |
| source "${KUBE_ROOT}/hack/lib/util.sh" |
| |
| if [[ "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" == "ubuntu" || "${NODE_OS_DISTRIBUTION}" == "custom" ]]; then |
| source "${KUBE_ROOT}/cluster/gce/${NODE_OS_DISTRIBUTION}/node-helper.sh" |
| else |
| echo "Cannot operate on cluster using node os distro: ${NODE_OS_DISTRIBUTION}" >&2 |
| exit 1 |
| fi |
| |
| source "${KUBE_ROOT}/cluster/gce/windows/node-helper.sh" |
| |
| if [[ "${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then |
| source "${KUBE_ROOT}/cluster/gce/${MASTER_OS_DISTRIBUTION}/master-helper.sh" |
| else |
| echo "Cannot operate on cluster using master os distro: ${MASTER_OS_DISTRIBUTION}" >&2 |
| exit 1 |
| fi |
| |
| if [[ ${NODE_LOCAL_SSDS:-} -ge 1 ]] && [[ -n ${NODE_LOCAL_SSDS_EXT:-} ]] ; then |
| echo -e "${color_red:-}Local SSD: Only one of NODE_LOCAL_SSDS and NODE_LOCAL_SSDS_EXT can be specified at once${color_norm:-}" >&2 |
| exit 2 |
| fi |
| |
| if [[ "${MASTER_OS_DISTRIBUTION}" == "gci" ]]; then |
| DEFAULT_GCI_PROJECT=google-containers |
| if [[ "${GCI_VERSION}" == "cos"* ]] || [[ "${MASTER_IMAGE_FAMILY}" == "cos"* ]]; then |
| DEFAULT_GCI_PROJECT=cos-cloud |
| fi |
| export MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-${DEFAULT_GCI_PROJECT}} |
| |
| # If the master image is not set, we use the latest image based on image |
| # family. |
| kube_master_image="${KUBE_GCE_MASTER_IMAGE:-${GCI_VERSION}}" |
| if [[ -z "${kube_master_image}" ]]; then |
| kube_master_image=$(gcloud compute images list --project="${MASTER_IMAGE_PROJECT}" --no-standard-images --filter="family:${MASTER_IMAGE_FAMILY}" --format 'value(name)') |
| fi |
| |
| echo "Using image: ${kube_master_image} from project: ${MASTER_IMAGE_PROJECT} as master image" >&2 |
| export MASTER_IMAGE="${kube_master_image}" |
| fi |
| |
| # Sets node image based on the specified os distro. Currently this function only |
| # supports gci and debian. |
| # |
| # Requires: |
| # NODE_OS_DISTRIBUTION |
| # Sets: |
| # DEFAULT_GCI_PROJECT |
| # NODE_IMAGE |
| # NODE_IMAGE_PROJECT |
| function set-linux-node-image() { |
| if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then |
| DEFAULT_GCI_PROJECT=google-containers |
| if [[ "${GCI_VERSION}" == "cos"* ]] || [[ "${NODE_IMAGE_FAMILY}" == "cos"* ]]; then |
| DEFAULT_GCI_PROJECT=cos-cloud |
| fi |
| |
| # If the node image is not set, we use the latest image based on image |
| # family. |
| # Otherwise, we respect whatever is set by the user. |
| NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-${DEFAULT_GCI_PROJECT}} |
| local kube_node_image |
| |
| kube_node_image="${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}}" |
| if [[ -z "${kube_node_image}" ]]; then |
| kube_node_image=$(gcloud compute images list --project="${NODE_IMAGE_PROJECT}" --no-standard-images --filter="family:${NODE_IMAGE_FAMILY}" --format 'value(name)') |
| fi |
| |
| echo "Using image: ${kube_node_image} from project: ${NODE_IMAGE_PROJECT} as node image" >&2 |
| export NODE_IMAGE="${kube_node_image}" |
| fi |
| } |
| |
| # Requires: |
| # WINDOWS_NODE_OS_DISTRIBUTION |
| # Sets: |
| # WINDOWS_NODE_IMAGE_PROJECT |
| # WINDOWS_NODE_IMAGE |
| function set-windows-node-image() { |
| WINDOWS_NODE_IMAGE_PROJECT="windows-cloud" |
| if [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win2019" ]]; then |
| WINDOWS_NODE_IMAGE="windows-server-2019-dc-core-v20210914" |
| elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win1909" ]]; then |
| WINDOWS_NODE_IMAGE="windows-server-1909-dc-core-v20210413" |
| elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION}" == "win2004" ]]; then |
| WINDOWS_NODE_IMAGE="windows-server-2004-dc-core-v20210914" |
| elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION,,}" == "win20h2" ]]; then |
| WINDOWS_NODE_IMAGE="windows-server-20h2-dc-core-v20210914" |
| elif [[ "${WINDOWS_NODE_OS_DISTRIBUTION,,}" == "win2022" ]]; then |
| WINDOWS_NODE_IMAGE="windows-server-2022-dc-core-v20220513" |
| else |
| echo "Unknown WINDOWS_NODE_OS_DISTRIBUTION ${WINDOWS_NODE_OS_DISTRIBUTION}" >&2 |
| exit 1 |
| fi |
| } |
| |
| set-linux-node-image |
| set-windows-node-image |
| |
| # Verify cluster autoscaler configuration. |
| if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then |
| if [[ -z $AUTOSCALER_MIN_NODES ]]; then |
| echo "AUTOSCALER_MIN_NODES not set." |
| exit 1 |
| fi |
| if [[ -z $AUTOSCALER_MAX_NODES ]]; then |
| echo "AUTOSCALER_MAX_NODES not set." |
| exit 1 |
| fi |
| fi |
| |
| # These prefixes must not be prefixes of each other, so that they can be used to |
| # detect mutually exclusive sets of nodes. |
| NODE_INSTANCE_PREFIX=${NODE_INSTANCE_PREFIX:-"${INSTANCE_PREFIX}-minion"} |
| WINDOWS_NODE_INSTANCE_PREFIX=${WINDOWS_NODE_INSTANCE_PREFIX:-"${INSTANCE_PREFIX}-windows-node"} |
| |
| # NODE_TAG (expected to be) defined by caller |
| # shellcheck disable=SC2153 |
| NODE_TAGS="${NODE_TAG}" |
| |
| ALLOCATE_NODE_CIDRS=true |
| PREEXISTING_NETWORK=false |
| PREEXISTING_NETWORK_MODE="" |
| |
| KUBE_PROMPT_FOR_UPDATE=${KUBE_PROMPT_FOR_UPDATE:-"n"} |
| # How long (in seconds) to wait for cluster initialization. |
| KUBE_CLUSTER_INITIALIZATION_TIMEOUT=${KUBE_CLUSTER_INITIALIZATION_TIMEOUT:-300} |
| |
| function join_csv() { |
| local IFS=','; echo "$*"; |
| } |
| |
| # This function returns the first string before the comma |
| function split_csv() { |
| echo "$*" | cut -d',' -f1 |
| } |
| |
| # Verify prereqs |
| function verify-prereqs() { |
| local cmd |
| |
| # we use openssl to generate certs |
| kube::util::test_openssl_installed |
| |
| # ensure a version supported by easyrsa is installed |
| if [ "$(openssl version | cut -d\ -f1)" == "LibreSSL" ]; then |
| echo "LibreSSL is not supported. Please ensure openssl points to an OpenSSL binary" |
| if [ "$(uname -s)" == "Darwin" ]; then |
| # We want this print just the way it is |
| # shellcheck disable=SC2016 |
| echo 'On macOS we recommend using homebrew and adding "$(brew --prefix openssl)/bin" to your PATH' |
| fi |
| exit 1 |
| fi |
| |
| # we use gcloud to create the cluster, gsutil to stage binaries and data |
| for cmd in gcloud gsutil; do |
| if ! which "${cmd}" >/dev/null; then |
| echo "Can't find ${cmd} in PATH, please fix and retry. The Google Cloud " >&2 |
| echo "SDK can be downloaded from https://cloud.google.com/sdk/." >&2 |
| exit 1 |
| fi |
| done |
| update-or-verify-gcloud |
| } |
| |
| # Use the gcloud defaults to find the project. If it is already set in the |
| # environment then go with that. |
| # |
| # Vars set: |
| # PROJECT |
| # NETWORK_PROJECT |
| # PROJECT_REPORTED |
| function detect-project() { |
| if [[ -z "${PROJECT-}" ]]; then |
| PROJECT=$(gcloud config list project --format 'value(core.project)') |
| fi |
| |
| NETWORK_PROJECT=${NETWORK_PROJECT:-${PROJECT}} |
| |
| if [[ -z "${PROJECT-}" ]]; then |
| echo "Could not detect Google Cloud Platform project. Set the default project using " >&2 |
| echo "'gcloud config set project <PROJECT>'" >&2 |
| exit 1 |
| fi |
| if [[ -z "${PROJECT_REPORTED-}" ]]; then |
| echo "Project: ${PROJECT}" >&2 |
| echo "Network Project: ${NETWORK_PROJECT}" >&2 |
| echo "Zone: ${ZONE}" >&2 |
| PROJECT_REPORTED=true |
| fi |
| } |
| |
| # Use gsutil to get the md5 hash for a particular tar |
| function gsutil_get_tar_md5() { |
| # location_tar could be local or in the cloud |
| # local tar_location example ./_output/release-tars/kubernetes-server-linux-amd64.tar.gz |
| # cloud tar_location example gs://kubernetes-staging-PROJECT/kubernetes-devel/kubernetes-server-linux-amd64.tar.gz |
| local -r tar_location=$1 |
| #parse the output and return the md5 hash |
| #the sed command at the end removes whitespace |
| local -r tar_md5=$(gsutil hash -h -m "${tar_location}" 2>/dev/null | grep "Hash (md5):" | awk -F ':' '{print $2}' | sed 's/^[[:space:]]*//g') |
| echo "${tar_md5}" |
| } |
| |
| # Copy a release tar and its accompanying hash. |
| function copy-to-staging() { |
| local -r staging_path=$1 |
| local -r gs_url=$2 |
| local -r tar=$3 |
| local -r hash=$4 |
| local -r basename_tar=$(basename "${tar}") |
| |
| #check whether this tar alread exists and has the same hash |
| #if it matches, then don't bother uploading it again |
| |
| #remote_tar_md5 checks the remote location for the existing tarball and its md5 |
| #staging_path example gs://kubernetes-staging-PROJECT/kubernetes-devel |
| #basename_tar example kubernetes-server-linux-amd64.tar.gz |
| local -r remote_tar_md5=$(gsutil_get_tar_md5 "${staging_path}/${basename_tar}") |
| if [[ -n ${remote_tar_md5} ]]; then |
| #local_tar_md5 checks the remote location for the existing tarball and its md5 hash |
| #tar example ./_output/release-tars/kubernetes-server-linux-amd64.tar.gz |
| local -r local_tar_md5=$(gsutil_get_tar_md5 "${tar}") |
| if [[ "${remote_tar_md5}" == "${local_tar_md5}" ]]; then |
| echo "+++ ${basename_tar} uploaded earlier, cloud and local file md5 match (md5 = ${local_tar_md5})" |
| return 0 |
| fi |
| fi |
| |
| echo "${hash}" > "${tar}.sha512" |
| gsutil -m -q -h "Cache-Control:private, max-age=0" cp "${tar}" "${tar}.sha512" "${staging_path}" |
| gsutil -m acl ch -g all:R "${gs_url}" "${gs_url}.sha512" >/dev/null 2>&1 || true |
| echo "+++ ${basename_tar} uploaded (sha512 = ${hash})" |
| } |
| |
| |
| # Given the cluster zone, return the list of regional GCS release |
| # bucket suffixes for the release in preference order. GCS doesn't |
| # give us an API for this, so we hardcode it. |
| # |
| # Assumed vars: |
| # RELEASE_REGION_FALLBACK |
| # REGIONAL_KUBE_ADDONS |
| # ZONE |
| # Vars set: |
| # PREFERRED_REGION |
| function set-preferred-region() { |
| case ${ZONE} in |
| asia-*) |
| PREFERRED_REGION=("asia-northeast1" "us-central1" "europe-west6") |
| ;; |
| europe-*) |
| PREFERRED_REGION=("europe-west6" "us-central1" "asia-northeast1") |
| ;; |
| *) |
| PREFERRED_REGION=("us-central1" "europe-west6" "asia-northeast1") |
| ;; |
| esac |
| |
| if [[ "${RELEASE_REGION_FALLBACK}" != "true" ]]; then |
| PREFERRED_REGION=( "${PREFERRED_REGION[0]}" ) |
| fi |
| } |
| |
| # Take the local tar files and upload them to Google Storage. They will then be |
| # downloaded by the master as part of the start up script for the master. |
| # |
| # Assumed vars: |
| # PROJECT |
| # SERVER_BINARY_TAR |
| # KUBE_MANIFESTS_TAR |
| # ZONE |
| # Vars set: |
| # SERVER_BINARY_TAR_URL |
| # SERVER_BINARY_TAR_HASH |
| # NODE_BINARY_TAR_URL |
| # NODE_BINARY_TAR_HASH |
| # KUBE_MANIFESTS_TAR_URL |
| # KUBE_MANIFESTS_TAR_HASH |
| function upload-tars() { |
| SERVER_BINARY_TAR_URL= |
| SERVER_BINARY_TAR_HASH= |
| NODE_BINARY_TAR_URL= |
| NODE_BINARY_TAR_HASH= |
| KUBE_MANIFESTS_TAR_URL= |
| KUBE_MANIFESTS_TAR_HASH= |
| |
| local project_hash |
| if which md5 > /dev/null 2>&1; then |
| project_hash=$(md5 -q -s "$PROJECT") |
| else |
| project_hash=$(echo -n "$PROJECT" | md5sum) |
| project_hash=${project_hash%%[[:blank:]]*} |
| fi |
| |
| # This requires 1 million projects before the probability of collision is 50% |
| # that's probably good enough for now :P |
| project_hash=${project_hash:0:10} |
| |
| set-preferred-region |
| |
| if [[ "${ENABLE_DOCKER_REGISTRY_CACHE:-}" == "true" ]]; then |
| DOCKER_REGISTRY_MIRROR_URL="https://mirror.gcr.io" |
| fi |
| |
| SERVER_BINARY_TAR_HASH=$(sha512sum-file "${SERVER_BINARY_TAR}") |
| |
| if [[ -n "${NODE_BINARY_TAR:-}" ]]; then |
| NODE_BINARY_TAR_HASH=$(sha512sum-file "${NODE_BINARY_TAR}") |
| fi |
| if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then |
| KUBE_MANIFESTS_TAR_HASH=$(sha512sum-file "${KUBE_MANIFESTS_TAR}") |
| fi |
| |
| local server_binary_tar_urls=() |
| local node_binary_tar_urls=() |
| |
| for region in "${PREFERRED_REGION[@]}"; do |
| suffix="-${region}" |
| local staging_bucket="gs://kubernetes-staging-${project_hash}${suffix}" |
| |
| # Ensure the buckets are created |
| if ! gsutil ls "${staging_bucket}" >/dev/null; then |
| echo "Creating ${staging_bucket}" |
| gsutil mb -l "${region}" -p "${PROJECT}" "${staging_bucket}" |
| fi |
| |
| local staging_path="${staging_bucket}/${INSTANCE_PREFIX}-devel" |
| |
| echo "+++ Staging tars to Google Storage: ${staging_path}" |
| local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}" |
| copy-to-staging "${staging_path}" "${server_binary_gs_url}" "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_HASH}" |
| |
| if [[ -n "${NODE_BINARY_TAR:-}" ]]; then |
| local node_binary_gs_url="${staging_path}/${NODE_BINARY_TAR##*/}" |
| copy-to-staging "${staging_path}" "${node_binary_gs_url}" "${NODE_BINARY_TAR}" "${NODE_BINARY_TAR_HASH}" |
| fi |
| |
| # Convert from gs:// URL to an https:// URL |
| server_binary_tar_urls+=("${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}") |
| if [[ -n "${NODE_BINARY_TAR:-}" ]]; then |
| node_binary_tar_urls+=("${node_binary_gs_url/gs:\/\//https://storage.googleapis.com/}") |
| fi |
| if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then |
| local kube_manifests_gs_url="${staging_path}/${KUBE_MANIFESTS_TAR##*/}" |
| copy-to-staging "${staging_path}" "${kube_manifests_gs_url}" "${KUBE_MANIFESTS_TAR}" "${KUBE_MANIFESTS_TAR_HASH}" |
| # Convert from gs:// URL to an https:// URL |
| kube_manifests_tar_urls+=("${kube_manifests_gs_url/gs:\/\//https://storage.googleapis.com/}") |
| fi |
| done |
| |
| SERVER_BINARY_TAR_URL=$(join_csv "${server_binary_tar_urls[@]}") |
| if [[ -n "${NODE_BINARY_TAR:-}" ]]; then |
| NODE_BINARY_TAR_URL=$(join_csv "${node_binary_tar_urls[@]}") |
| fi |
| if [[ -n "${KUBE_MANIFESTS_TAR:-}" ]]; then |
| KUBE_MANIFESTS_TAR_URL=$(join_csv "${kube_manifests_tar_urls[@]}") |
| fi |
| } |
| |
| # Detect Linux and Windows nodes created in the instance group. |
| # |
| # Assumed vars: |
| # NODE_INSTANCE_PREFIX |
| # WINDOWS_NODE_INSTANCE_PREFIX |
| # Vars set: |
| # NODE_NAMES |
| # INSTANCE_GROUPS |
| # WINDOWS_NODE_NAMES |
| # WINDOWS_INSTANCE_GROUPS |
| function detect-node-names() { |
| detect-project |
| INSTANCE_GROUPS=() |
| kube::util::read-array INSTANCE_GROUPS < <(gcloud compute instance-groups managed list \ |
| --project "${PROJECT}" \ |
| --filter "name ~ '${NODE_INSTANCE_PREFIX}-.+' AND zone:(${ZONE})" \ |
| --format='value(name)' || true) |
| WINDOWS_INSTANCE_GROUPS=() |
| kube::util::read-array WINDOWS_INSTANCE_GROUPS < <(gcloud compute instance-groups managed list \ |
| --project "${PROJECT}" \ |
| --filter "name ~ '${WINDOWS_NODE_INSTANCE_PREFIX}-.+' AND zone:(${ZONE})" \ |
| --format='value(name)' || true) |
| |
| NODE_NAMES=() |
| if [[ -n "${INSTANCE_GROUPS[*]:-}" ]]; then |
| for group in "${INSTANCE_GROUPS[@]}"; do |
| kube::util::read-array NODE_NAMES < <(gcloud compute instance-groups managed list-instances \ |
| "${group}" --zone "${ZONE}" --project "${PROJECT}" \ |
| --format='value(name)') |
| done |
| fi |
| # Add heapster node name to the list too (if it exists). |
| if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then |
| NODE_NAMES+=("${NODE_INSTANCE_PREFIX}-heapster") |
| fi |
| export NODE_NAMES |
| WINDOWS_NODE_NAMES=() |
| if [[ -n "${WINDOWS_INSTANCE_GROUPS[*]:-}" ]]; then |
| for group in "${WINDOWS_INSTANCE_GROUPS[@]}"; do |
| kube::util::read-array WINDOWS_NODE_NAMES < <(gcloud compute instance-groups managed \ |
| list-instances "${group}" --zone "${ZONE}" --project "${PROJECT}" \ |
| --format='value(name)') |
| done |
| fi |
| export WINDOWS_NODE_NAMES |
| |
| echo "INSTANCE_GROUPS=${INSTANCE_GROUPS[*]:-}" >&2 |
| echo "NODE_NAMES=${NODE_NAMES[*]:-}" >&2 |
| } |
| |
| # Detect the information about the minions |
| # |
| # Assumed vars: |
| # ZONE |
| # Vars set: |
| # NODE_NAMES |
| # KUBE_NODE_IP_ADDRESSES (array) |
| function detect-nodes() { |
| detect-project |
| detect-node-names |
| KUBE_NODE_IP_ADDRESSES=() |
| for (( i=0; i<${#NODE_NAMES[@]}; i++)); do |
| local node_ip |
| node_ip=$(gcloud compute instances describe --project "${PROJECT}" --zone "${ZONE}" \ |
| "${NODE_NAMES[$i]}" --format='value(networkInterfaces[0].accessConfigs[0].natIP)') |
| if [[ -z "${node_ip-}" ]] ; then |
| echo "Did not find ${NODE_NAMES[$i]}" >&2 |
| else |
| echo "Found ${NODE_NAMES[$i]} at ${node_ip}" |
| KUBE_NODE_IP_ADDRESSES+=("${node_ip}") |
| fi |
| done |
| if [[ -z "${KUBE_NODE_IP_ADDRESSES-}" ]]; then |
| echo "Could not detect Kubernetes minion nodes. Make sure you've launched a cluster with 'kube-up.sh'" >&2 |
| exit 1 |
| fi |
| } |
| |
| # Detect the IP for the master |
| # |
| # Assumed vars: |
| # MASTER_NAME |
| # ZONE |
| # REGION |
| # Vars set: |
| # KUBE_MASTER |
| # KUBE_MASTER_IP |
| function detect-master() { |
| detect-project |
| KUBE_MASTER=${MASTER_NAME} |
| echo "Trying to find master named '${MASTER_NAME}'" >&2 |
| if [[ -z "${KUBE_MASTER_IP-}" ]]; then |
| local master_address_name="${MASTER_NAME}-ip" |
| echo "Looking for address '${master_address_name}'" >&2 |
| if ! KUBE_MASTER_IP=$(gcloud compute addresses describe "${master_address_name}" \ |
| --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') || \ |
| [[ -z "${KUBE_MASTER_IP-}" ]]; then |
| echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'" >&2 |
| exit 1 |
| fi |
| fi |
| if [[ -z "${KUBE_MASTER_INTERNAL_IP-}" ]] && [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then |
| local master_address_name="${MASTER_NAME}-internal-ip" |
| echo "Looking for address '${master_address_name}'" >&2 |
| if ! KUBE_MASTER_INTERNAL_IP=$(gcloud compute addresses describe "${master_address_name}" \ |
| --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') || \ |
| [[ -z "${KUBE_MASTER_INTERNAL_IP-}" ]]; then |
| echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'" >&2 |
| exit 1 |
| fi |
| fi |
| echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP; internal IP: ${KUBE_MASTER_INTERNAL_IP:-(not set)})" >&2 |
| } |
| |
| function load-or-gen-kube-bearertoken() { |
| if [[ -n "${KUBE_CONTEXT:-}" ]]; then |
| get-kubeconfig-bearertoken |
| fi |
| if [[ -z "${KUBE_BEARER_TOKEN:-}" ]]; then |
| gen-kube-bearertoken |
| fi |
| } |
| |
| # Figure out which binary use on the server and assure it is available. |
| # If KUBE_VERSION is specified use binaries specified by it, otherwise |
| # use local dev binaries. |
| # |
| # Assumed vars: |
| # KUBE_VERSION |
| # KUBE_RELEASE_VERSION_REGEX |
| # KUBE_CI_VERSION_REGEX |
| # Vars set: |
| # KUBE_TAR_HASH |
| # SERVER_BINARY_TAR_URL |
| # SERVER_BINARY_TAR_HASH |
| function tars_from_version() { |
| local sha512sum="" |
| if which sha512sum >/dev/null 2>&1; then |
| sha512sum="sha512sum" |
| else |
| sha512sum="shasum -a512" |
| fi |
| |
| if [[ -z "${KUBE_VERSION-}" ]]; then |
| find-release-tars |
| upload-tars |
| elif [[ ${KUBE_VERSION} =~ ${KUBE_RELEASE_VERSION_REGEX} ]]; then |
| SERVER_BINARY_TAR_URL="https://dl.k8s.io/release/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz" |
| # TODO: Clean this up. |
| KUBE_MANIFESTS_TAR_URL="${SERVER_BINARY_TAR_URL/server-linux-amd64/manifests}" |
| KUBE_MANIFESTS_TAR_HASH=$(curl -L "${KUBE_MANIFESTS_TAR_URL}" --silent --show-error | ${sha512sum}) |
| KUBE_MANIFESTS_TAR_HASH=${KUBE_MANIFESTS_TAR_HASH%%[[:blank:]]*} |
| elif [[ ${KUBE_VERSION} =~ ${KUBE_CI_VERSION_REGEX} ]]; then |
| SERVER_BINARY_TAR_URL="https://storage.googleapis.com/k8s-release-dev/ci/${KUBE_VERSION}/kubernetes-server-linux-amd64.tar.gz" |
| # TODO: Clean this up. |
| KUBE_MANIFESTS_TAR_URL="${SERVER_BINARY_TAR_URL/server-linux-amd64/manifests}" |
| KUBE_MANIFESTS_TAR_HASH=$(curl "${KUBE_MANIFESTS_TAR_URL}" --silent --show-error | ${sha512sum}) |
| KUBE_MANIFESTS_TAR_HASH=${KUBE_MANIFESTS_TAR_HASH%%[[:blank:]]*} |
| else |
| echo "Version doesn't match regexp" >&2 |
| exit 1 |
| fi |
| if ! SERVER_BINARY_TAR_HASH=$(curl -Ss --fail "${SERVER_BINARY_TAR_URL}.sha512"); then |
| echo "Failure trying to curl release .sha512" |
| fi |
| |
| if ! curl -Ss --head "${SERVER_BINARY_TAR_URL}" >&/dev/null; then |
| echo "Can't find release at ${SERVER_BINARY_TAR_URL}" >&2 |
| exit 1 |
| fi |
| } |
| |
| # Reads kube-env metadata from master |
| # |
| # Assumed vars: |
| # KUBE_MASTER |
| # PROJECT |
| # ZONE |
| function get-master-env() { |
| # TODO(zmerlynn): Make this more reliable with retries. |
| gcloud compute --project "${PROJECT}" ssh --zone "${ZONE}" "${KUBE_MASTER}" --command \ |
| "curl --fail --silent -H 'Metadata-Flavor: Google' \ |
| 'http://metadata/computeMetadata/v1/instance/attributes/kube-env'" 2>/dev/null |
| gcloud compute --project "${PROJECT}" ssh --zone "${ZONE}" "${KUBE_MASTER}" --command \ |
| "curl --fail --silent -H 'Metadata-Flavor: Google' \ |
| 'http://metadata/computeMetadata/v1/instance/attributes/kube-master-certs'" 2>/dev/null |
| } |
| |
| # Quote something appropriate for a yaml string. |
| # |
| # TODO(zmerlynn): Note that this function doesn't so much "quote" as |
| # "strip out quotes", and we really should be using a YAML library for |
| # this, but PyYAML isn't shipped by default, and *rant rant rant ... SIGH* |
| function yaml-quote { |
| echo "${@:-}" | sed -e "s/'/''/g;s/^/'/i;s/$/'/i" |
| } |
| |
| # Writes the cluster location into a temporary file. |
| # Assumed vars |
| # ZONE |
| function write-cluster-location { |
| cat >"${KUBE_TEMP}/cluster-location.txt" << EOF |
| ${ZONE} |
| EOF |
| } |
| |
| # Writes the cluster name into a temporary file. |
| # Assumed vars |
| # CLUSTER_NAME |
| function write-cluster-name { |
| cat >"${KUBE_TEMP}/cluster-name.txt" << EOF |
| ${CLUSTER_NAME} |
| EOF |
| } |
| |
| function write-master-env { |
| # If the user requested that the master be part of the cluster, set the |
| # environment variable to program the master kubelet to register itself. |
| if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" && -z "${KUBELET_APISERVER:-}" ]]; then |
| KUBELET_APISERVER="${MASTER_NAME}" |
| fi |
| if [[ -z "${KUBERNETES_MASTER_NAME:-}" ]]; then |
| KUBERNETES_MASTER_NAME="${MASTER_NAME}" |
| fi |
| |
| construct-linux-kubelet-flags "master" |
| build-linux-kube-env true "${KUBE_TEMP}/master-kube-env.yaml" |
| build-kubelet-config true "linux" "${KUBE_TEMP}/master-kubelet-config.yaml" |
| build-kube-master-certs "${KUBE_TEMP}/kube-master-certs.yaml" |
| } |
| |
| function write-linux-node-env { |
| if [[ -z "${KUBERNETES_MASTER_NAME:-}" ]]; then |
| KUBERNETES_MASTER_NAME="${MASTER_NAME}" |
| fi |
| |
| construct-linux-kubelet-flags "heapster" |
| build-linux-kube-env false "${KUBE_TEMP}/heapster-kube-env.yaml" |
| construct-linux-kubelet-flags "node" |
| build-linux-kube-env false "${KUBE_TEMP}/node-kube-env.yaml" |
| build-kubelet-config false "linux" "${KUBE_TEMP}/node-kubelet-config.yaml" |
| } |
| |
| function write-windows-node-env { |
| construct-windows-kubelet-flags |
| construct-windows-kubeproxy-flags |
| build-windows-kube-env "${KUBE_TEMP}/windows-node-kube-env.yaml" |
| build-kubelet-config false "windows" "${KUBE_TEMP}/windows-node-kubelet-config.yaml" |
| } |
| |
| function build-linux-node-labels { |
| local node_type=$1 |
| local node_labels="" |
| if [[ "${KUBE_PROXY_DAEMONSET:-}" == "true" && "${node_type}" != "master" ]]; then |
| # Add kube-proxy daemonset label to node to avoid situation during cluster |
| # upgrade/downgrade when there are two instances of kube-proxy running on a node. |
| node_labels="node.kubernetes.io/kube-proxy-ds-ready=true" |
| fi |
| if [[ -n "${NODE_LABELS:-}" ]]; then |
| node_labels="${node_labels:+${node_labels},}${NODE_LABELS}" |
| fi |
| if [[ -n "${NON_MASTER_NODE_LABELS:-}" && "${node_type}" != "master" ]]; then |
| node_labels="${node_labels:+${node_labels},}${NON_MASTER_NODE_LABELS}" |
| fi |
| if [[ -n "${MASTER_NODE_LABELS:-}" && "${node_type}" == "master" ]]; then |
| node_labels="${node_labels:+${node_labels},}${MASTER_NODE_LABELS}" |
| fi |
| echo "$node_labels" |
| } |
| |
| function build-windows-node-labels { |
| local node_labels="" |
| if [[ -n "${WINDOWS_NODE_LABELS:-}" ]]; then |
| node_labels="${node_labels:+${node_labels},}${WINDOWS_NODE_LABELS}" |
| fi |
| if [[ -n "${WINDOWS_NON_MASTER_NODE_LABELS:-}" ]]; then |
| node_labels="${node_labels:+${node_labels},}${WINDOWS_NON_MASTER_NODE_LABELS}" |
| fi |
| echo "$node_labels" |
| } |
| |
| # yaml-map-string-stringarray converts the encoded structure to yaml format, and echoes the result |
| # under the provided name. If the encoded structure is empty, echoes nothing. |
| # 1: name to be output in yaml |
| # 2: encoded map-string-string (which may contain duplicate keys - resulting in map-string-stringarray) |
| # 3: key-value separator (defaults to ':') |
| # 4: item separator (defaults to ',') |
| function yaml-map-string-stringarray { |
| declare -r name="${1}" |
| declare -r encoded="${2}" |
| declare -r kv_sep="${3:-:}" |
| declare -r item_sep="${4:-,}" |
| |
| declare -a pairs # indexed array |
| declare -A map # associative array |
| IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep |
| for pair in "${pairs[@]}"; do |
| declare key |
| declare value |
| IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep |
| map[$key]="${map[$key]+${map[$key]}${item_sep}}${value}" # append values from duplicate keys |
| done |
| # only output if there is a non-empty map |
| if [[ ${#map[@]} -gt 0 ]]; then |
| echo "${name}:" |
| for k in "${!map[@]}"; do |
| echo " ${k}:" |
| declare -a values |
| IFS="${item_sep}" read -ra values <<<"${map[$k]}" |
| for val in "${values[@]}"; do |
| # declare across two lines so errexit can catch failures |
| declare v |
| v=$(yaml-quote "${val}") |
| echo " - ${v}" |
| done |
| done |
| fi |
| } |
| |
| # yaml-map-string-string converts the encoded structure to yaml format, and echoes the result |
| # under the provided name. If the encoded structure is empty, echoes nothing. |
| # 1: name to be output in yaml |
| # 2: encoded map-string-string (no duplicate keys) |
| # 3: bool, whether to yaml-quote the value string in the output (defaults to true) |
| # 4: key-value separator (defaults to ':') |
| # 5: item separator (defaults to ',') |
| function yaml-map-string-string { |
| declare -r name="${1}" |
| declare -r encoded="${2}" |
| declare -r quote_val_string="${3:-true}" |
| declare -r kv_sep="${4:-:}" |
| declare -r item_sep="${5:-,}" |
| |
| declare -a pairs # indexed array |
| declare -A map # associative array |
| IFS="${item_sep}" read -ra pairs <<<"${encoded}" # split on item_sep # TODO(mtaufen): try quoting this too |
| for pair in "${pairs[@]}"; do |
| declare key |
| declare value |
| IFS="${kv_sep}" read -r key value <<<"${pair}" # split on kv_sep |
| map[$key]="${value}" # add to associative array |
| done |
| # only output if there is a non-empty map |
| if [[ ${#map[@]} -gt 0 ]]; then |
| echo "${name}:" |
| for k in "${!map[@]}"; do |
| if [[ "${quote_val_string}" == "true" ]]; then |
| # declare across two lines so errexit can catch failures |
| declare v |
| v=$(yaml-quote "${map[$k]}") |
| echo " ${k}: ${v}" |
| else |
| echo " ${k}: ${map[$k]}" |
| fi |
| done |
| fi |
| } |
| |
| # Returns kubelet flags used on both Linux and Windows nodes. |
| function construct-common-kubelet-flags { |
| local flags="${KUBELET_TEST_LOG_LEVEL:-"--v=2"} ${KUBELET_TEST_ARGS:-}" |
| flags+=" --cloud-provider=${CLOUD_PROVIDER_FLAG:-external}" |
| # TODO(mtaufen): ROTATE_CERTIFICATES seems unused; delete it? |
| if [[ -n "${ROTATE_CERTIFICATES:-}" ]]; then |
| flags+=" --rotate-certificates=true" |
| fi |
| if [[ -n "${MAX_PODS_PER_NODE:-}" ]]; then |
| flags+=" --max-pods=${MAX_PODS_PER_NODE}" |
| fi |
| echo "$flags" |
| } |
| |
| # Sets KUBELET_ARGS with the kubelet flags for Linux nodes. |
| # $1: if 'true', we're rendering flags for a master, else a node |
| function construct-linux-kubelet-flags { |
| local node_type="$1" |
| local flags |
| flags="$(construct-common-kubelet-flags)" |
| # Keep in sync with CONTAINERIZED_MOUNTER_HOME in configure-helper.sh |
| flags+=" --experimental-mounter-path=/home/kubernetes/containerized_mounter/mounter" |
| # Keep in sync with the mkdir command in configure-helper.sh (until the TODO is resolved) |
| flags+=" --cert-dir=/var/lib/kubelet/pki/" |
| |
| # If ENABLE_AUTH_PROVIDER_GCP is set to true, kubelet is enabled to use out-of-tree auth |
| # credential provider instead of in-tree auth credential provider. |
| # https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider |
| if [[ "${ENABLE_AUTH_PROVIDER_GCP:-true}" == "true" ]]; then |
| # Keep the values of --image-credential-provider-config and --image-credential-provider-bin-dir |
| # in sync with value of auth_config_file and auth_provider_dir set in install-auth-provider-gcp function |
| # in gci/configure.sh. |
| flags+=" --image-credential-provider-config=${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}" |
| flags+=" --image-credential-provider-bin-dir=${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}" |
| fi |
| |
| if [[ "${node_type}" == "master" ]]; then |
| flags+=" ${MASTER_KUBELET_TEST_ARGS:-}" |
| if [[ "${REGISTER_MASTER_KUBELET:-false}" == "true" ]]; then |
| #TODO(mikedanese): allow static pods to start before creating a client |
| #flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" |
| #flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig" |
| flags+=" --register-with-taints=node-role.kubernetes.io/control-plane=:NoSchedule" |
| flags+=" --kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" |
| flags+=" --register-schedulable=false" |
| fi |
| if [[ "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]]; then |
| # Configure the file path for host dns configuration |
| # as ubuntu uses systemd-resolved |
| flags+=" --resolv-conf=/run/systemd/resolve/resolv.conf" |
| fi |
| else # For nodes |
| flags+=" ${NODE_KUBELET_TEST_ARGS:-}" |
| flags+=" --bootstrap-kubeconfig=/var/lib/kubelet/bootstrap-kubeconfig" |
| flags+=" --kubeconfig=/var/lib/kubelet/kubeconfig" |
| if [[ "${node_type}" == "heapster" ]]; then |
| flags+=" ${HEAPSTER_KUBELET_TEST_ARGS:-}" |
| fi |
| if [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then |
| # Configure the file path for host dns configuration |
| # as ubuntu uses systemd-resolved |
| flags+=" --resolv-conf=/run/systemd/resolve/resolv.conf" |
| fi |
| fi |
| flags+=" --volume-plugin-dir=${VOLUME_PLUGIN_DIR}" |
| local node_labels |
| node_labels="$(build-linux-node-labels "${node_type}")" |
| if [[ -n "${node_labels:-}" ]]; then |
| flags+=" --node-labels=${node_labels}" |
| fi |
| if [[ -n "${NODE_TAINTS:-}" ]]; then |
| flags+=" --register-with-taints=${NODE_TAINTS}" |
| fi |
| |
| CONTAINER_RUNTIME_ENDPOINT=${KUBE_CONTAINER_RUNTIME_ENDPOINT:-unix:///run/containerd/containerd.sock} |
| flags+=" --container-runtime-endpoint=${CONTAINER_RUNTIME_ENDPOINT}" |
| |
| if [[ "${CONTAINER_RUNTIME_ENDPOINT}" =~ /containerd.sock$ ]]; then |
| flags+=" --runtime-cgroups=/system.slice/containerd.service" |
| fi |
| |
| KUBELET_ARGS="${flags}" |
| } |
| |
| # Sets KUBELET_ARGS with the kubelet flags for Windows nodes. |
| # Note that to configure flags with explicit empty string values, we can't escape |
| # double-quotes, because they still break sc.exe after expansion in the |
| # binPath parameter, and single-quotes get parsed as characters instead of |
| # string delimiters. |
| function construct-windows-kubelet-flags { |
| local flags |
| flags="$(construct-common-kubelet-flags)" |
| |
| # Note: NODE_KUBELET_TEST_ARGS is empty in typical kube-up runs. |
| flags+=" ${NODE_KUBELET_TEST_ARGS:-}" |
| |
| local node_labels |
| node_labels="$(build-windows-node-labels)" |
| if [[ -n "${node_labels:-}" ]]; then |
| flags+=" --node-labels=${node_labels}" |
| fi |
| |
| # Concatenate common and windows-only node taints and apply them. |
| local node_taints="${NODE_TAINTS:-}" |
| if [[ -n "${node_taints}" && -n "${WINDOWS_NODE_TAINTS:-}" ]]; then |
| node_taints+=":${WINDOWS_NODE_TAINTS}" |
| else |
| node_taints="${WINDOWS_NODE_TAINTS:-}" |
| fi |
| if [[ -n "${node_taints}" ]]; then |
| flags+=" --register-with-taints=${node_taints}" |
| fi |
| |
| # Many of these flags were adapted from |
| # https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1. |
| flags+=" --config=${WINDOWS_KUBELET_CONFIG_FILE}" |
| flags+=" --kubeconfig=${WINDOWS_KUBECONFIG_FILE}" |
| |
| # The directory where the TLS certs are located. |
| flags+=" --cert-dir=${WINDOWS_PKI_DIR}" |
| flags+=" --pod-manifest-path=${WINDOWS_MANIFESTS_DIR}" |
| |
| # Configure kubelet to run as a windows service. |
| flags+=" --windows-service=true" |
| |
| # Configure the file path for host dns configuration |
| flags+=" --resolv-conf=${WINDOWS_CNI_DIR}\hostdns.conf" |
| |
| # Both --cgroups-per-qos and --enforce-node-allocatable should be disabled on |
| # windows; the latter requires the former to be enabled to work. |
| flags+=" --cgroups-per-qos=false --enforce-node-allocatable=" |
| |
| # Turn off kernel memory cgroup notification. |
| flags+=" --kernel-memcg-notification=false" |
| |
| WINDOWS_CONTAINER_RUNTIME_ENDPOINT=${KUBE_WINDOWS_CONTAINER_RUNTIME_ENDPOINT:-npipe:////./pipe/containerd-containerd} |
| flags+=" --container-runtime-endpoint=${WINDOWS_CONTAINER_RUNTIME_ENDPOINT}" |
| |
| # If ENABLE_AUTH_PROVIDER_GCP is set to true, kubelet is enabled to use out-of-tree auth |
| # credential provider. https://kubernetes.io/docs/tasks/kubelet-credential-provider/kubelet-credential-provider |
| if [[ "${ENABLE_AUTH_PROVIDER_GCP:-true}" == "true" ]]; then |
| flags+=" --image-credential-provider-config=${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}" |
| flags+=" --image-credential-provider-bin-dir=${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}" |
| fi |
| |
| KUBELET_ARGS="${flags}" |
| } |
| |
| function construct-windows-kubeproxy-flags { |
| local flags="" |
| |
| # Use the same log level as the Kubelet during tests. |
| flags+=" ${KUBELET_TEST_LOG_LEVEL:-"--v=2"}" |
| |
| # Windows uses kernelspace proxymode |
| flags+=" --proxy-mode=kernelspace" |
| |
| # Configure kube-proxy to run as a windows service. |
| flags+=" --windows-service=true" |
| |
| # Enabling Windows DSR mode unlocks newer network features and reduces |
| # port usage for services. |
| # https://techcommunity.microsoft.com/t5/networking-blog/direct-server-return-dsr-in-a-nutshell/ba-p/693710 |
| if [[ "${WINDOWS_ENABLE_DSR:-}" == "true" ]]; then |
| flags+=" --feature-gates=WinDSR=true --enable-dsr=true " |
| fi |
| |
| # Configure flags with explicit empty string values. We can't escape |
| # double-quotes, because they still break sc.exe after expansion in the |
| # binPath parameter, and single-quotes get parsed as characters instead |
| # of string delimiters. |
| |
| KUBEPROXY_ARGS="${flags}" |
| } |
| |
| # $1: if 'true', we're rendering config for a master, else a node |
| function build-kubelet-config { |
| local master="$1" |
| local os="$2" |
| local file="$3" |
| |
| rm -f "${file}" |
| { |
| print-common-kubelet-config |
| if [[ "${master}" == "true" ]]; then |
| print-master-kubelet-config |
| else |
| print-common-node-kubelet-config |
| if [[ "${os}" == "linux" ]]; then |
| print-linux-node-kubelet-config |
| elif [[ "${os}" == "windows" ]]; then |
| print-windows-node-kubelet-config |
| else |
| echo "Unknown OS ${os}" >&2 |
| exit 1 |
| fi |
| fi |
| } > "${file}" |
| } |
| |
| # cat the Kubelet config yaml in common between masters, linux nodes, and |
| # windows nodes |
| function print-common-kubelet-config { |
| declare quoted_dns_server_ip |
| declare quoted_dns_domain |
| quoted_dns_server_ip=$(yaml-quote "${DNS_SERVER_IP}") |
| quoted_dns_domain=$(yaml-quote "${DNS_DOMAIN}") |
| cat <<EOF |
| kind: KubeletConfiguration |
| apiVersion: kubelet.config.k8s.io/v1beta1 |
| cgroupRoot: / |
| clusterDNS: |
| - ${quoted_dns_server_ip} |
| clusterDomain: ${quoted_dns_domain} |
| readOnlyPort: 10255 |
| EOF |
| |
| # Note: ENABLE_MANIFEST_URL is used by GKE. |
| # TODO(mtaufen): remove this since it's not used in kubernetes/kubernetes nor |
| # kubernetes/test-infra. |
| if [[ "${ENABLE_MANIFEST_URL:-}" == "true" ]]; then |
| declare quoted_manifest_url |
| quoted_manifest_url=$(yaml-quote "${MANIFEST_URL}") |
| cat <<EOF |
| staticPodURL: ${quoted_manifest_url} |
| EOF |
| yaml-map-string-stringarray 'staticPodURLHeader' "${MANIFEST_URL_HEADER}" |
| fi |
| |
| if [[ -n "${EVICTION_HARD:-}" ]]; then |
| yaml-map-string-string 'evictionHard' "${EVICTION_HARD}" true '<' |
| fi |
| |
| if [[ -n "${FEATURE_GATES:-}" ]]; then |
| yaml-map-string-string 'featureGates' "${FEATURE_GATES}" false '=' |
| fi |
| } |
| |
| # cat the Kubelet config yaml for masters |
| function print-master-kubelet-config { |
| cat <<EOF |
| enableDebuggingHandlers: ${MASTER_KUBELET_ENABLE_DEBUGGING_HANDLERS:-false} |
| hairpinMode: none |
| staticPodPath: /etc/kubernetes/manifests |
| authentication: |
| webhook: |
| enabled: false |
| anonymous: |
| enabled: true |
| authorization: |
| mode: AlwaysAllow |
| EOF |
| if [[ "${REGISTER_MASTER_KUBELET:-false}" == "false" ]]; then |
| # Note: Standalone mode is used by GKE |
| declare quoted_master_ip_range |
| quoted_master_ip_range=$(yaml-quote "${MASTER_IP_RANGE}") |
| cat <<EOF |
| podCidr: ${quoted_master_ip_range} |
| EOF |
| fi |
| } |
| |
| # cat the Kubelet config yaml in common between linux nodes and windows nodes |
| function print-common-node-kubelet-config { |
| cat <<EOF |
| enableDebuggingHandlers: ${KUBELET_ENABLE_DEBUGGING_HANDLERS:-true} |
| EOF |
| if [[ "${HAIRPIN_MODE:-}" == "promiscuous-bridge" ]] || \ |
| [[ "${HAIRPIN_MODE:-}" == "hairpin-veth" ]] || \ |
| [[ "${HAIRPIN_MODE:-}" == "none" ]]; then |
| declare quoted_hairpin_mode |
| quoted_hairpin_mode=$(yaml-quote "${HAIRPIN_MODE}") |
| cat <<EOF |
| hairpinMode: ${quoted_hairpin_mode} |
| EOF |
| fi |
| } |
| |
| # cat the Kubelet config yaml for linux nodes |
| function print-linux-node-kubelet-config { |
| # Keep authentication.x509.clientCAFile in sync with CA_CERT_BUNDLE_PATH in configure-helper.sh |
| cat <<EOF |
| staticPodPath: /etc/kubernetes/manifests |
| authentication: |
| x509: |
| clientCAFile: /etc/srv/kubernetes/pki/ca-certificates.crt |
| EOF |
| } |
| |
| # cat the Kubelet config yaml for windows nodes |
| function print-windows-node-kubelet-config { |
| # Notes: |
| # - We don't run any static pods on Windows nodes yet. |
| |
| # TODO(mtaufen): Does it make any sense to set eviction thresholds for inodes |
| # on Windows? |
| |
| # TODO(pjh, mtaufen): It may make sense to use a different hairpin mode on |
| # Windows. We're currently using hairpin-veth, but |
| # https://github.com/Microsoft/SDN/blob/master/Kubernetes/windows/start-kubelet.ps1#L121 |
| # uses promiscuous-bridge. |
| |
| # TODO(pjh, mtaufen): Does cgroupRoot make sense for Windows? |
| |
| # Keep authentication.x509.clientCAFile in sync with CA_CERT_BUNDLE_PATH in |
| # k8s-node-setup.psm1. |
| cat <<EOF |
| authentication: |
| x509: |
| clientCAFile: '${WINDOWS_CA_FILE}' |
| EOF |
| } |
| |
| function build-kube-master-certs { |
| local file=$1 |
| rm -f "$file" |
| cat >"$file" <<EOF |
| KUBEAPISERVER_CERT: $(yaml-quote "${KUBEAPISERVER_CERT_BASE64:-}") |
| KUBEAPISERVER_KEY: $(yaml-quote "${KUBEAPISERVER_KEY_BASE64:-}") |
| CA_KEY: $(yaml-quote "${CA_KEY_BASE64:-}") |
| AGGREGATOR_CA_KEY: $(yaml-quote "${AGGREGATOR_CA_KEY_BASE64:-}") |
| REQUESTHEADER_CA_CERT: $(yaml-quote "${REQUESTHEADER_CA_CERT_BASE64:-}") |
| PROXY_CLIENT_CERT: $(yaml-quote "${PROXY_CLIENT_CERT_BASE64:-}") |
| PROXY_CLIENT_KEY: $(yaml-quote "${PROXY_CLIENT_KEY_BASE64:-}") |
| ETCD_APISERVER_CA_KEY: $(yaml-quote "${ETCD_APISERVER_CA_KEY_BASE64:-}") |
| ETCD_APISERVER_CA_CERT: $(yaml-quote "${ETCD_APISERVER_CA_CERT_BASE64:-}") |
| ETCD_APISERVER_SERVER_KEY: $(yaml-quote "${ETCD_APISERVER_SERVER_KEY_BASE64:-}") |
| ETCD_APISERVER_SERVER_CERT: $(yaml-quote "${ETCD_APISERVER_SERVER_CERT_BASE64:-}") |
| ETCD_APISERVER_CLIENT_KEY: $(yaml-quote "${ETCD_APISERVER_CLIENT_KEY_BASE64:-}") |
| ETCD_APISERVER_CLIENT_CERT: $(yaml-quote "${ETCD_APISERVER_CLIENT_CERT_BASE64:-}") |
| CLOUD_PVL_ADMISSION_CA_KEY: $(yaml-quote "${CLOUD_PVL_ADMISSION_CA_KEY_BASE64:-}") |
| CLOUD_PVL_ADMISSION_CA_CERT: $(yaml-quote "${CLOUD_PVL_ADMISSION_CA_CERT_BASE64:-}") |
| CLOUD_PVL_ADMISSION_CERT: $(yaml-quote "${CLOUD_PVL_ADMISSION_CERT_BASE64:-}") |
| CLOUD_PVL_ADMISSION_KEY: $(yaml-quote "${CLOUD_PVL_ADMISSION_KEY_BASE64:-}") |
| KONNECTIVITY_SERVER_CA_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CA_KEY_BASE64:-}") |
| KONNECTIVITY_SERVER_CA_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CA_CERT_BASE64:-}") |
| KONNECTIVITY_SERVER_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CERT_BASE64:-}") |
| KONNECTIVITY_SERVER_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_KEY_BASE64:-}") |
| KONNECTIVITY_SERVER_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_CERT_BASE64:-}") |
| KONNECTIVITY_SERVER_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_SERVER_CLIENT_KEY_BASE64:-}") |
| KONNECTIVITY_AGENT_CA_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CA_KEY_BASE64:-}") |
| KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}") |
| KONNECTIVITY_AGENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CERT_BASE64:-}") |
| KONNECTIVITY_AGENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_KEY_BASE64:-}") |
| KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}") |
| KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}") |
| EOF |
| } |
| |
| # $1: if 'true', we're building a master yaml, else a node |
| function build-linux-kube-env { |
| local master="$1" |
| local file="$2" |
| |
| local server_binary_tar_url=$SERVER_BINARY_TAR_URL |
| local kube_manifests_tar_url="${KUBE_MANIFESTS_TAR_URL:-}" |
| if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "ubuntu" ]] || \ |
| [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "ubuntu" || "${NODE_OS_DISTRIBUTION}" == "custom") ]]; then |
| # TODO: Support fallback .tar.gz settings on Container Linux |
| server_binary_tar_url=$(split_csv "${SERVER_BINARY_TAR_URL}") |
| kube_manifests_tar_url=$(split_csv "${KUBE_MANIFESTS_TAR_URL}") |
| fi |
| |
| rm -f "$file" |
| cat >"$file" <<EOF |
| CLUSTER_NAME: $(yaml-quote "${CLUSTER_NAME}") |
| ENV_TIMESTAMP: $(yaml-quote "$(date -u +%Y-%m-%dT%T%z)") |
| INSTANCE_PREFIX: $(yaml-quote "${INSTANCE_PREFIX}") |
| NODE_INSTANCE_PREFIX: $(yaml-quote "${NODE_INSTANCE_PREFIX}") |
| NODE_TAGS: $(yaml-quote "${NODE_TAGS:-}") |
| NODE_NETWORK: $(yaml-quote "${NETWORK:-}") |
| NODE_SUBNETWORK: $(yaml-quote "${SUBNETWORK:-}") |
| CLUSTER_IP_RANGE: $(yaml-quote "${CLUSTER_IP_RANGE:-10.244.0.0/16}") |
| SERVER_BINARY_TAR_URL: $(yaml-quote "${server_binary_tar_url}") |
| SERVER_BINARY_TAR_HASH: $(yaml-quote "${SERVER_BINARY_TAR_HASH}") |
| PROJECT_ID: $(yaml-quote "${PROJECT}") |
| NETWORK_PROJECT_ID: $(yaml-quote "${NETWORK_PROJECT}") |
| SERVICE_CLUSTER_IP_RANGE: $(yaml-quote "${SERVICE_CLUSTER_IP_RANGE}") |
| KUBERNETES_MASTER_NAME: $(yaml-quote "${KUBERNETES_MASTER_NAME}") |
| ALLOCATE_NODE_CIDRS: $(yaml-quote "${ALLOCATE_NODE_CIDRS:-false}") |
| ENABLE_METRICS_SERVER: $(yaml-quote "${ENABLE_METRICS_SERVER:-false}") |
| ENABLE_METADATA_AGENT: $(yaml-quote "${ENABLE_METADATA_AGENT:-none}") |
| METADATA_AGENT_CPU_REQUEST: $(yaml-quote "${METADATA_AGENT_CPU_REQUEST:-}") |
| METADATA_AGENT_MEMORY_REQUEST: $(yaml-quote "${METADATA_AGENT_MEMORY_REQUEST:-}") |
| METADATA_AGENT_CLUSTER_LEVEL_CPU_REQUEST: $(yaml-quote "${METADATA_AGENT_CLUSTER_LEVEL_CPU_REQUEST:-}") |
| METADATA_AGENT_CLUSTER_LEVEL_MEMORY_REQUEST: $(yaml-quote "${METADATA_AGENT_CLUSTER_LEVEL_MEMORY_REQUEST:-}") |
| DOCKER_REGISTRY_MIRROR_URL: $(yaml-quote "${DOCKER_REGISTRY_MIRROR_URL:-}") |
| ENABLE_L7_LOADBALANCING: $(yaml-quote "${ENABLE_L7_LOADBALANCING:-none}") |
| ENABLE_CLUSTER_LOGGING: $(yaml-quote "${ENABLE_CLUSTER_LOGGING:-false}") |
| ENABLE_AUTH_PROVIDER_GCP: $(yaml-quote "${ENABLE_AUTH_PROVIDER_GCP:-true}") |
| ENABLE_NODE_PROBLEM_DETECTOR: $(yaml-quote "${ENABLE_NODE_PROBLEM_DETECTOR:-none}") |
| NODE_PROBLEM_DETECTOR_VERSION: $(yaml-quote "${NODE_PROBLEM_DETECTOR_VERSION:-}") |
| NODE_PROBLEM_DETECTOR_TAR_HASH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TAR_HASH:-}") |
| NODE_PROBLEM_DETECTOR_RELEASE_PATH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_RELEASE_PATH:-}") |
| NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS: $(yaml-quote "${NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS:-}") |
| CNI_STORAGE_URL_BASE: $(yaml-quote "${CNI_STORAGE_URL_BASE:-}") |
| CNI_TAR_PREFIX: $(yaml-quote "${CNI_TAR_PREFIX:-}") |
| CNI_VERSION: $(yaml-quote "${CNI_VERSION:-}") |
| CNI_HASH: $(yaml-quote "${CNI_HASH:-}") |
| ENABLE_NODE_LOGGING: $(yaml-quote "${ENABLE_NODE_LOGGING:-false}") |
| LOGGING_DESTINATION: $(yaml-quote "${LOGGING_DESTINATION:-}") |
| ELASTICSEARCH_LOGGING_REPLICAS: $(yaml-quote "${ELASTICSEARCH_LOGGING_REPLICAS:-}") |
| ENABLE_CLUSTER_DNS: $(yaml-quote "${ENABLE_CLUSTER_DNS:-false}") |
| CLUSTER_DNS_CORE_DNS: $(yaml-quote "${CLUSTER_DNS_CORE_DNS:-true}") |
| ENABLE_NODELOCAL_DNS: $(yaml-quote "${ENABLE_NODELOCAL_DNS:-false}") |
| DNS_SERVER_IP: $(yaml-quote "${DNS_SERVER_IP:-}") |
| LOCAL_DNS_IP: $(yaml-quote "${LOCAL_DNS_IP:-}") |
| DNS_DOMAIN: $(yaml-quote "${DNS_DOMAIN:-}") |
| DNS_MEMORY_LIMIT: $(yaml-quote "${DNS_MEMORY_LIMIT:-}") |
| ENABLE_DNS_HORIZONTAL_AUTOSCALER: $(yaml-quote "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-false}") |
| KUBE_PROXY_DAEMONSET: $(yaml-quote "${KUBE_PROXY_DAEMONSET:-false}") |
| KUBE_PROXY_TOKEN: $(yaml-quote "${KUBE_PROXY_TOKEN:-}") |
| KUBE_PROXY_MODE: $(yaml-quote "${KUBE_PROXY_MODE:-iptables}") |
| DETECT_LOCAL_MODE: $(yaml-quote "${DETECT_LOCAL_MODE:-}") |
| NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TOKEN:-}") |
| ADMISSION_CONTROL: $(yaml-quote "${ADMISSION_CONTROL:-}") |
| MASTER_IP_RANGE: $(yaml-quote "${MASTER_IP_RANGE}") |
| RUNTIME_CONFIG: $(yaml-quote "${RUNTIME_CONFIG}") |
| CA_CERT: $(yaml-quote "${CA_CERT_BASE64:-}") |
| KUBELET_CERT: $(yaml-quote "${KUBELET_CERT_BASE64:-}") |
| KUBELET_KEY: $(yaml-quote "${KUBELET_KEY_BASE64:-}") |
| NETWORK_PROVIDER: $(yaml-quote "${NETWORK_PROVIDER:-}") |
| NETWORK_POLICY_PROVIDER: $(yaml-quote "${NETWORK_POLICY_PROVIDER:-}") |
| HAIRPIN_MODE: $(yaml-quote "${HAIRPIN_MODE:-}") |
| E2E_STORAGE_TEST_ENVIRONMENT: $(yaml-quote "${E2E_STORAGE_TEST_ENVIRONMENT:-}") |
| KUBE_DOCKER_REGISTRY: $(yaml-quote "${KUBE_DOCKER_REGISTRY:-}") |
| KUBE_ADDON_REGISTRY: $(yaml-quote "${KUBE_ADDON_REGISTRY:-}") |
| MULTIZONE: $(yaml-quote "${MULTIZONE:-}") |
| MULTIMASTER: $(yaml-quote "${MULTIMASTER:-}") |
| NON_MASQUERADE_CIDR: $(yaml-quote "${NON_MASQUERADE_CIDR:-}") |
| ENABLE_DEFAULT_STORAGE_CLASS: $(yaml-quote "${ENABLE_DEFAULT_STORAGE_CLASS:-}") |
| ENABLE_VOLUME_SNAPSHOTS: $(yaml-quote "${ENABLE_VOLUME_SNAPSHOTS:-}") |
| ENABLE_APISERVER_ADVANCED_AUDIT: $(yaml-quote "${ENABLE_APISERVER_ADVANCED_AUDIT:-}") |
| ENABLE_APISERVER_DYNAMIC_AUDIT: $(yaml-quote "${ENABLE_APISERVER_DYNAMIC_AUDIT:-}") |
| ENABLE_CACHE_MUTATION_DETECTOR: $(yaml-quote "${ENABLE_CACHE_MUTATION_DETECTOR:-false}") |
| ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR: $(yaml-quote "${ENABLE_KUBE_WATCHLIST_INCONSISTENCY_DETECTOR:-false}") |
| ENABLE_PATCH_CONVERSION_DETECTOR: $(yaml-quote "${ENABLE_PATCH_CONVERSION_DETECTOR:-false}") |
| ADVANCED_AUDIT_POLICY: $(yaml-quote "${ADVANCED_AUDIT_POLICY:-}") |
| ADVANCED_AUDIT_BACKEND: $(yaml-quote "${ADVANCED_AUDIT_BACKEND:-log}") |
| ADVANCED_AUDIT_TRUNCATING_BACKEND: $(yaml-quote "${ADVANCED_AUDIT_TRUNCATING_BACKEND:-true}") |
| ADVANCED_AUDIT_LOG_MODE: $(yaml-quote "${ADVANCED_AUDIT_LOG_MODE:-}") |
| ADVANCED_AUDIT_LOG_BUFFER_SIZE: $(yaml-quote "${ADVANCED_AUDIT_LOG_BUFFER_SIZE:-}") |
| ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE: $(yaml-quote "${ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE:-}") |
| ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT: $(yaml-quote "${ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT:-}") |
| ADVANCED_AUDIT_LOG_THROTTLE_QPS: $(yaml-quote "${ADVANCED_AUDIT_LOG_THROTTLE_QPS:-}") |
| ADVANCED_AUDIT_LOG_THROTTLE_BURST: $(yaml-quote "${ADVANCED_AUDIT_LOG_THROTTLE_BURST:-}") |
| ADVANCED_AUDIT_LOG_INITIAL_BACKOFF: $(yaml-quote "${ADVANCED_AUDIT_LOG_INITIAL_BACKOFF:-}") |
| ADVANCED_AUDIT_WEBHOOK_MODE: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_MODE:-}") |
| ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE:-}") |
| ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_SIZE:-}") |
| ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_MAX_BATCH_WAIT:-}") |
| ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_QPS:-}") |
| ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST:-}") |
| ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF: $(yaml-quote "${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF:-}") |
| GCE_API_ENDPOINT: $(yaml-quote "${GCE_API_ENDPOINT:-}") |
| GCE_GLBC_IMAGE: $(yaml-quote "${GCE_GLBC_IMAGE:-}") |
| CUSTOM_INGRESS_YAML: | |
| ${CUSTOM_INGRESS_YAML//\'/\'\'} |
| ENABLE_NODE_JOURNAL: $(yaml-quote "${ENABLE_NODE_JOURNAL:-false}") |
| PROMETHEUS_TO_SD_ENDPOINT: $(yaml-quote "${PROMETHEUS_TO_SD_ENDPOINT:-}") |
| PROMETHEUS_TO_SD_PREFIX: $(yaml-quote "${PROMETHEUS_TO_SD_PREFIX:-}") |
| ENABLE_PROMETHEUS_TO_SD: $(yaml-quote "${ENABLE_PROMETHEUS_TO_SD:-false}") |
| DISABLE_PROMETHEUS_TO_SD_IN_DS: $(yaml-quote "${DISABLE_PROMETHEUS_TO_SD_IN_DS:-false}") |
| CONTAINER_RUNTIME_ENDPOINT: $(yaml-quote "${CONTAINER_RUNTIME_ENDPOINT:-}") |
| CONTAINER_RUNTIME_NAME: $(yaml-quote "${CONTAINER_RUNTIME_NAME:-}") |
| CONTAINER_RUNTIME_TEST_HANDLER: $(yaml-quote "${CONTAINER_RUNTIME_TEST_HANDLER:-}") |
| CONTAINERD_INFRA_CONTAINER: $(yaml-quote "${CONTAINER_INFRA_CONTAINER:-}") |
| UBUNTU_INSTALL_CONTAINERD_VERSION: $(yaml-quote "${UBUNTU_INSTALL_CONTAINERD_VERSION:-}") |
| UBUNTU_INSTALL_RUNC_VERSION: $(yaml-quote "${UBUNTU_INSTALL_RUNC_VERSION:-}") |
| COS_INSTALL_CONTAINERD_VERSION: $(yaml-quote "${COS_INSTALL_CONTAINERD_VERSION:-}") |
| COS_INSTALL_RUNC_VERSION: $(yaml-quote "${COS_INSTALL_RUNC_VERSION:-}") |
| NODE_LOCAL_SSDS_EXT: $(yaml-quote "${NODE_LOCAL_SSDS_EXT:-}") |
| NODE_LOCAL_SSDS_EPHEMERAL: $(yaml-quote "${NODE_LOCAL_SSDS_EPHEMERAL:-}") |
| LOAD_IMAGE_COMMAND: $(yaml-quote "${LOAD_IMAGE_COMMAND:-}") |
| ZONE: $(yaml-quote "${ZONE}") |
| REGION: $(yaml-quote "${REGION}") |
| VOLUME_PLUGIN_DIR: $(yaml-quote "${VOLUME_PLUGIN_DIR}") |
| KUBELET_ARGS: $(yaml-quote "${KUBELET_ARGS}") |
| REQUIRE_METADATA_KUBELET_CONFIG_FILE: $(yaml-quote true) |
| ENABLE_NETD: $(yaml-quote "${ENABLE_NETD:-false}") |
| CUSTOM_NETD_YAML: | |
| ${CUSTOM_NETD_YAML//\'/\'\'} |
| CUSTOM_CALICO_NODE_DAEMONSET_YAML: | |
| ${CUSTOM_CALICO_NODE_DAEMONSET_YAML//\'/\'\'} |
| CUSTOM_TYPHA_DEPLOYMENT_YAML: | |
| ${CUSTOM_TYPHA_DEPLOYMENT_YAML//\'/\'\'} |
| CONCURRENT_SERVICE_SYNCS: $(yaml-quote "${CONCURRENT_SERVICE_SYNCS:-}") |
| AUTH_PROVIDER_GCP_STORAGE_PATH: $(yaml-quote "${AUTH_PROVIDER_GCP_STORAGE_PATH}") |
| AUTH_PROVIDER_GCP_VERSION: $(yaml-quote "${AUTH_PROVIDER_GCP_VERSION}") |
| AUTH_PROVIDER_GCP_LINUX_BIN_DIR: $(yaml-quote "${AUTH_PROVIDER_GCP_LINUX_BIN_DIR}") |
| AUTH_PROVIDER_GCP_LINUX_CONF_FILE: $(yaml-quote "${AUTH_PROVIDER_GCP_LINUX_CONF_FILE}") |
| EOF |
| if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || \ |
| [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "gci" ]] || \ |
| [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "cos" ]] || \ |
| [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "cos" ]]; then |
| cat >>"$file" <<EOF |
| REMOUNT_VOLUME_PLUGIN_DIR: $(yaml-quote "${REMOUNT_VOLUME_PLUGIN_DIR:-true}") |
| EOF |
| fi |
| if [[ "${master}" == "false" ]]; then |
| cat >>"$file" <<EOF |
| KONNECTIVITY_AGENT_CA_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CA_CERT_BASE64:-}") |
| KONNECTIVITY_AGENT_CLIENT_KEY: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_KEY_BASE64:-}") |
| KONNECTIVITY_AGENT_CLIENT_CERT: $(yaml-quote "${KONNECTIVITY_AGENT_CLIENT_CERT_BASE64:-}") |
| EOF |
| fi |
| if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]; then |
| cat >>"$file" <<EOF |
| KUBE_APISERVER_REQUEST_TIMEOUT: $(yaml-quote "${KUBE_APISERVER_REQUEST_TIMEOUT}") |
| EOF |
| fi |
| if [ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]; then |
| cat >>"$file" <<EOF |
| TERMINATED_POD_GC_THRESHOLD: $(yaml-quote "${TERMINATED_POD_GC_THRESHOLD}") |
| EOF |
| fi |
| if [[ "${master}" == "true" && ("${MASTER_OS_DISTRIBUTION}" == "trusty" || "${MASTER_OS_DISTRIBUTION}" == "gci" || "${MASTER_OS_DISTRIBUTION}" == "ubuntu") ]] || \ |
| [[ "${master}" == "false" && ("${NODE_OS_DISTRIBUTION}" == "trusty" || "${NODE_OS_DISTRIBUTION}" == "gci" || "${NODE_OS_DISTRIBUTION}" = "ubuntu" || "${NODE_OS_DISTRIBUTION}" = "custom") ]] ; then |
| cat >>"$file" <<EOF |
| KUBE_MANIFESTS_TAR_URL: $(yaml-quote "${kube_manifests_tar_url}") |
| KUBE_MANIFESTS_TAR_HASH: $(yaml-quote "${KUBE_MANIFESTS_TAR_HASH}") |
| EOF |
| fi |
| if [ -n "${TEST_CLUSTER:-}" ]; then |
| cat >>"$file" <<EOF |
| TEST_CLUSTER: $(yaml-quote "${TEST_CLUSTER}") |
| EOF |
| fi |
| if [ -n "${DOCKER_TEST_LOG_LEVEL:-}" ]; then |
| cat >>"$file" <<EOF |
| DOCKER_TEST_LOG_LEVEL: $(yaml-quote "${DOCKER_TEST_LOG_LEVEL}") |
| EOF |
| fi |
| if [ -n "${DOCKER_LOG_DRIVER:-}" ]; then |
| cat >>"$file" <<EOF |
| DOCKER_LOG_DRIVER: $(yaml-quote "${DOCKER_LOG_DRIVER}") |
| EOF |
| fi |
| if [ -n "${DOCKER_LOG_MAX_SIZE:-}" ]; then |
| cat >>"$file" <<EOF |
| DOCKER_LOG_MAX_SIZE: $(yaml-quote "${DOCKER_LOG_MAX_SIZE}") |
| EOF |
| fi |
| if [ -n "${DOCKER_LOG_MAX_FILE:-}" ]; then |
| cat >>"$file" <<EOF |
| DOCKER_LOG_MAX_FILE: $(yaml-quote "${DOCKER_LOG_MAX_FILE}") |
| EOF |
| fi |
| if [ -n "${CLOUD_PROVIDER_FLAG:-}" ]; then |
| cat >>"$file" <<EOF |
| CLOUD_PROVIDER_FLAG: $(yaml-quote "${CLOUD_PROVIDER_FLAG}") |
| EOF |
| fi |
| if [ -n "${FEATURE_GATES:-}" ]; then |
| cat >>"$file" <<EOF |
| FEATURE_GATES: $(yaml-quote "${FEATURE_GATES}") |
| EOF |
| fi |
| if [ -n "${RUN_CONTROLLERS:-}" ]; then |
| cat >>"$file" <<EOF |
| RUN_CONTROLLERS: $(yaml-quote "${RUN_CONTROLLERS}") |
| EOF |
| fi |
| if [ -n "${RUN_CCM_CONTROLLERS:-}" ]; then |
| cat >>"$file" <<EOF |
| RUN_CCM_CONTROLLERS: $(yaml-quote "${RUN_CCM_CONTROLLERS}") |
| EOF |
| fi |
| if [ -n "${PROVIDER_VARS:-}" ]; then |
| local var_name |
| local var_value |
| |
| for var_name in ${PROVIDER_VARS}; do |
| eval "local var_value=\$(yaml-quote \${${var_name}})" |
| cat >>"$file" <<EOF |
| ${var_name}: ${var_value} |
| EOF |
| done |
| fi |
| |
| if [[ "${master}" == "true" ]]; then |
| # Master-only env vars. |
| cat >>"$file" <<EOF |
| KUBERNETES_MASTER: $(yaml-quote 'true') |
| KUBE_USER: $(yaml-quote "${KUBE_USER}") |
| KUBE_PASSWORD: $(yaml-quote "${KUBE_PASSWORD}") |
| KUBE_BEARER_TOKEN: $(yaml-quote "${KUBE_BEARER_TOKEN}") |
| MASTER_CERT: $(yaml-quote "${MASTER_CERT_BASE64:-}") |
| MASTER_KEY: $(yaml-quote "${MASTER_KEY_BASE64:-}") |
| KUBECFG_CERT: $(yaml-quote "${KUBECFG_CERT_BASE64:-}") |
| KUBECFG_KEY: $(yaml-quote "${KUBECFG_KEY_BASE64:-}") |
| KUBELET_APISERVER: $(yaml-quote "${KUBELET_APISERVER:-}") |
| NUM_NODES: $(yaml-quote "${NUM_NODES}") |
| STORAGE_BACKEND: $(yaml-quote "${STORAGE_BACKEND:-etcd3}") |
| STORAGE_MEDIA_TYPE: $(yaml-quote "${STORAGE_MEDIA_TYPE:-}") |
| ENABLE_GARBAGE_COLLECTOR: $(yaml-quote "${ENABLE_GARBAGE_COLLECTOR:-}") |
| ENABLE_LEGACY_ABAC: $(yaml-quote "${ENABLE_LEGACY_ABAC:-}") |
| MASTER_ADVERTISE_ADDRESS: $(yaml-quote "${MASTER_ADVERTISE_ADDRESS:-}") |
| ETCD_CA_KEY: $(yaml-quote "${ETCD_CA_KEY_BASE64:-}") |
| ETCD_CA_CERT: $(yaml-quote "${ETCD_CA_CERT_BASE64:-}") |
| ETCD_PEER_KEY: $(yaml-quote "${ETCD_PEER_KEY_BASE64:-}") |
| ETCD_PEER_CERT: $(yaml-quote "${ETCD_PEER_CERT_BASE64:-}") |
| SERVICEACCOUNT_ISSUER: $(yaml-quote "${SERVICEACCOUNT_ISSUER:-}") |
| KUBECTL_PRUNE_WHITELIST_OVERRIDE: $(yaml-quote "${KUBECTL_PRUNE_WHITELIST_OVERRIDE:-}") |
| CCM_FEATURE_GATES: $(yaml-quote "${CCM_FEATURE_GATES:-}") |
| KUBE_SCHEDULER_RUNASUSER: 2001 |
| KUBE_SCHEDULER_RUNASGROUP: 2001 |
| KUBE_ADDON_MANAGER_RUNASUSER: 2002 |
| KUBE_ADDON_MANAGER_RUNASGROUP: 2002 |
| KUBE_CONTROLLER_MANAGER_RUNASUSER: 2003 |
| KUBE_CONTROLLER_MANAGER_RUNASGROUP: 2003 |
| KUBE_API_SERVER_RUNASUSER: 2004 |
| KUBE_API_SERVER_RUNASGROUP: 2004 |
| KUBE_PKI_READERS_GROUP: 2005 |
| ETCD_RUNASUSER: 2006 |
| ETCD_RUNASGROUP: 2006 |
| KUBE_POD_LOG_READERS_GROUP: 2007 |
| KONNECTIVITY_SERVER_RUNASUSER: 2008 |
| KONNECTIVITY_SERVER_RUNASGROUP: 2008 |
| KONNECTIVITY_SERVER_SOCKET_WRITER_GROUP: 2008 |
| CLOUD_CONTROLLER_MANAGER_RUNASUSER: 2009 |
| CLOUD_CONTROLLER_MANAGER_RUNASGROUP: 2009 |
| CLUSTER_AUTOSCALER_RUNASUSER: 2010 |
| CLUSTER_AUTOSCALER_RUNASGROUP: 2010 |
| |
| EOF |
| # KUBE_APISERVER_REQUEST_TIMEOUT_SEC (if set) controls the --request-timeout |
| # flag |
| if [ -n "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC:-}" ]; then |
| cat >>"$file" <<EOF |
| KUBE_APISERVER_REQUEST_TIMEOUT_SEC: $(yaml-quote "${KUBE_APISERVER_REQUEST_TIMEOUT_SEC}") |
| EOF |
| fi |
| # KUBE_APISERVER_GODEBUG (if set) controls the value of GODEBUG env var for kube-apiserver. |
| if [ -n "${KUBE_APISERVER_GODEBUG:-}" ]; then |
| cat >>"$file" <<EOF |
| KUBE_APISERVER_GODEBUG: $(yaml-quote "${KUBE_APISERVER_GODEBUG}") |
| EOF |
| fi |
| # ETCD_IMAGE (if set) allows to use a custom etcd image. |
| if [ -n "${ETCD_IMAGE:-}" ]; then |
| cat >>"$file" <<EOF |
| ETCD_IMAGE: $(yaml-quote "${ETCD_IMAGE}") |
| EOF |
| fi |
| # ETCD_DOCKER_REPOSITORY (if set) allows to use a custom etcd docker repository to pull the etcd image from. |
| if [ -n "${ETCD_DOCKER_REPOSITORY:-}" ]; then |
| cat >>"$file" <<EOF |
| ETCD_DOCKER_REPOSITORY: $(yaml-quote "${ETCD_DOCKER_REPOSITORY}") |
| EOF |
| fi |
| # ETCD_VERSION (if set) allows you to use custom version of etcd. |
| # The main purpose of using it may be rollback of etcd v3 API, |
| # where we need 3.0.* image, but are rolling back to 2.3.7. |
| if [ -n "${ETCD_VERSION:-}" ]; then |
| cat >>"$file" <<EOF |
| ETCD_VERSION: $(yaml-quote "${ETCD_VERSION}") |
| EOF |
| fi |
| if [ -n "${ETCD_HOSTNAME:-}" ]; then |
| cat >>"$file" <<EOF |
| ETCD_HOSTNAME: $(yaml-quote "${ETCD_HOSTNAME}") |
| EOF |
| fi |
| if [ -n "${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC:-}" ]; then |
| cat >>"$file" <<EOF |
| ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC: $(yaml-quote "${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC}") |
| EOF |
| fi |
| if [ -n "${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-}" ]; then |
| cat >>"$file" <<EOF |
| KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC: $(yaml-quote "${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC}") |
| EOF |
| fi |
| if [ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]; then |
| cat >>"$file" <<EOF |
| ETCD_COMPACTION_INTERVAL_SEC: $(yaml-quote "${ETCD_COMPACTION_INTERVAL_SEC}") |
| EOF |
| fi |
| if [ -n "${ETCD_QUOTA_BACKEND_BYTES:-}" ]; then |
| cat >>"$file" <<EOF |
| ETCD_QUOTA_BACKEND_BYTES: $(yaml-quote "${ETCD_QUOTA_BACKEND_BYTES}") |
| EOF |
| fi |
| if [ -n "${ETCD_EXTRA_ARGS:-}" ]; then |
| cat >>"$file" <<EOF |
| ETCD_EXTRA_ARGS: $(yaml-quote "${ETCD_EXTRA_ARGS}") |
| EOF |
| fi |
| if [ -n "${ETCD_SERVERS:-}" ]; then |
| cat >>"$file" <<EOF |
| ETCD_SERVERS: $(yaml-quote "${ETCD_SERVERS}") |
| EOF |
| fi |
| if [ -n "${ETCD_SERVERS_OVERRIDES:-}" ]; then |
| cat >>"$file" <<EOF |
| ETCD_SERVERS_OVERRIDES: $(yaml-quote "${ETCD_SERVERS_OVERRIDES}") |
| EOF |
| fi |
| if [ -n "${APISERVER_TEST_ARGS:-}" ]; then |
| cat >>"$file" <<EOF |
| APISERVER_TEST_ARGS: $(yaml-quote "${APISERVER_TEST_ARGS}") |
| EOF |
| fi |
| if [ -n "${CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then |
| cat >>"$file" <<EOF |
| CONTROLLER_MANAGER_TEST_ARGS: $(yaml-quote "${CONTROLLER_MANAGER_TEST_ARGS}") |
| EOF |
| fi |
| if [ -n "${KUBE_CONTROLLER_MANAGER_TEST_ARGS:-}" ]; then |
| cat >>"$file" <<EOF |
| KUBE_CONTROLLER_MANAGER_TEST_ARGS: $(yaml-quote "${KUBE_CONTROLLER_MANAGER_TEST_ARGS}") |
| EOF |
| fi |
| if [ -n "${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-}" ]; then |
| cat >>"$file" <<EOF |
| CONTROLLER_MANAGER_TEST_LOG_LEVEL: $(yaml-quote "${CONTROLLER_MANAGER_TEST_LOG_LEVEL}") |
| EOF |
| fi |
| if [ -n "${SCHEDULER_TEST_ARGS:-}" ]; then |
| cat >>"$file" <<EOF |
| SCHEDULER_TEST_ARGS: $(yaml-quote "${SCHEDULER_TEST_ARGS}") |
| EOF |
| fi |
| if [ -n "${SCHEDULER_TEST_LOG_LEVEL:-}" ]; then |
| cat >>"$file" <<EOF |
| SCHEDULER_TEST_LOG_LEVEL: $(yaml-quote "${SCHEDULER_TEST_LOG_LEVEL}") |
| EOF |
| fi |
| if [ -n "${INITIAL_ETCD_CLUSTER:-}" ]; then |
| cat >>"$file" <<EOF |
| INITIAL_ETCD_CLUSTER: $(yaml-quote "${INITIAL_ETCD_CLUSTER}") |
| EOF |
| fi |
| if [ -n "${INITIAL_ETCD_CLUSTER_STATE:-}" ]; then |
| cat >>"$file" <<EOF |
| INITIAL_ETCD_CLUSTER_STATE: $(yaml-quote "${INITIAL_ETCD_CLUSTER_STATE}") |
| EOF |
| fi |
| if [ -n "${CLUSTER_SIGNING_DURATION:-}" ]; then |
| cat >>"$file" <<EOF |
| CLUSTER_SIGNING_DURATION: $(yaml-quote "${CLUSTER_SIGNING_DURATION}") |
| EOF |
| fi |
| if [[ "${NODE_ACCELERATORS:-}" == *"type=nvidia"* ]]; then |
| cat >>"$file" <<EOF |
| ENABLE_NVIDIA_GPU_DEVICE_PLUGIN: $(yaml-quote "true") |
| EOF |
| fi |
| if [ -n "${ADDON_MANAGER_LEADER_ELECTION:-}" ]; then |
| cat >>"$file" <<EOF |
| ADDON_MANAGER_LEADER_ELECTION: $(yaml-quote "${ADDON_MANAGER_LEADER_ELECTION}") |
| EOF |
| fi |
| if [ -n "${API_SERVER_TEST_LOG_LEVEL:-}" ]; then |
| cat >>"$file" <<EOF |
| API_SERVER_TEST_LOG_LEVEL: $(yaml-quote "${API_SERVER_TEST_LOG_LEVEL}") |
| EOF |
| fi |
| if [ -n "${ETCD_LISTEN_CLIENT_IP:-}" ]; then |
| cat >>"$file" <<EOF |
| ETCD_LISTEN_CLIENT_IP: $(yaml-quote "${ETCD_LISTEN_CLIENT_IP}") |
| EOF |
| fi |
| if [ -n "${ETCD_PROGRESS_NOTIFY_INTERVAL:-}" ]; then |
| cat >>"$file" <<EOF |
| ETCD_PROGRESS_NOTIFY_INTERVAL: $(yaml-quote "${ETCD_PROGRESS_NOTIFY_INTERVAL}") |
| EOF |
| fi |
| |
| else |
| # Node-only env vars. |
| cat >>"$file" <<EOF |
| KUBERNETES_MASTER: $(yaml-quote "false") |
| EXTRA_DOCKER_OPTS: $(yaml-quote "${EXTRA_DOCKER_OPTS:-}") |
| EOF |
| if [ -n "${KUBEPROXY_TEST_ARGS:-}" ]; then |
| cat >>"$file" <<EOF |
| KUBEPROXY_TEST_ARGS: $(yaml-quote "${KUBEPROXY_TEST_ARGS}") |
| EOF |
| fi |
| if [ -n "${KUBEPROXY_TEST_LOG_LEVEL:-}" ]; then |
| cat >>"$file" <<EOF |
| KUBEPROXY_TEST_LOG_LEVEL: $(yaml-quote "${KUBEPROXY_TEST_LOG_LEVEL}") |
| EOF |
| fi |
| fi |
| if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then |
| cat >>"$file" <<EOF |
| ENABLE_CLUSTER_AUTOSCALER: $(yaml-quote "${ENABLE_CLUSTER_AUTOSCALER}") |
| AUTOSCALER_MIG_CONFIG: $(yaml-quote "${AUTOSCALER_MIG_CONFIG}") |
| AUTOSCALER_EXPANDER_CONFIG: $(yaml-quote "${AUTOSCALER_EXPANDER_CONFIG}") |
| EOF |
| if [[ "${master}" == "false" ]]; then |
| # TODO(kubernetes/autoscaler#718): AUTOSCALER_ENV_VARS is a hotfix for cluster autoscaler, |
| # which reads the kube-env to determine the shape of a node and was broken by #60020. |
| # This should be removed as soon as a more reliable source of information is available! |
| local node_labels |
| local node_taints |
| local autoscaler_env_vars |
| node_labels="$(build-linux-node-labels node)" |
| node_taints="${NODE_TAINTS:-}" |
| autoscaler_env_vars="node_labels=${node_labels};node_taints=${node_taints}" |
| cat >>"$file" <<EOF |
| AUTOSCALER_ENV_VARS: $(yaml-quote "${autoscaler_env_vars}") |
| EOF |
| fi |
| fi |
| if [ -n "${SCHEDULING_ALGORITHM_PROVIDER:-}" ]; then |
| cat >>"$file" <<EOF |
| SCHEDULING_ALGORITHM_PROVIDER: $(yaml-quote "${SCHEDULING_ALGORITHM_PROVIDER}") |
| EOF |
| fi |
| if [ -n "${MAX_PODS_PER_NODE:-}" ]; then |
| cat >>"$file" <<EOF |
| MAX_PODS_PER_NODE: $(yaml-quote "${MAX_PODS_PER_NODE}") |
| EOF |
| fi |
| if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then |
| cat >>"$file" <<EOF |
| PREPARE_KONNECTIVITY_SERVICE: $(yaml-quote "${PREPARE_KONNECTIVITY_SERVICE}") |
| EOF |
| fi |
| if [[ "${EGRESS_VIA_KONNECTIVITY:-false}" == "true" ]]; then |
| cat >>"$file" <<EOF |
| EGRESS_VIA_KONNECTIVITY: $(yaml-quote "${EGRESS_VIA_KONNECTIVITY}") |
| EOF |
| fi |
| if [[ "${RUN_KONNECTIVITY_PODS:-false}" == "true" ]]; then |
| cat >>"$file" <<EOF |
| RUN_KONNECTIVITY_PODS: $(yaml-quote "${RUN_KONNECTIVITY_PODS}") |
| EOF |
| fi |
| if [[ -n "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE:-}" ]]; then |
| cat >>"$file" <<EOF |
| KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE: $(yaml-quote "${KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE}") |
| EOF |
| fi |
| } |
| |
| |
| function build-windows-kube-env { |
| local file="$1" |
| # For now the Windows kube-env is a superset of the Linux kube-env. |
| build-linux-kube-env false "$file" |
| |
| cat >>"$file" <<EOF |
| WINDOWS_NODE_INSTANCE_PREFIX: $(yaml-quote "${WINDOWS_NODE_INSTANCE_PREFIX}") |
| NODE_BINARY_TAR_URL: $(yaml-quote "${NODE_BINARY_TAR_URL}") |
| NODE_BINARY_TAR_HASH: $(yaml-quote "${NODE_BINARY_TAR_HASH}") |
| CSI_PROXY_STORAGE_PATH: $(yaml-quote "${CSI_PROXY_STORAGE_PATH}") |
| CSI_PROXY_VERSION: $(yaml-quote "${CSI_PROXY_VERSION}") |
| CSI_PROXY_FLAGS: $(yaml-quote "${CSI_PROXY_FLAGS}") |
| ENABLE_CSI_PROXY: $(yaml-quote "${ENABLE_CSI_PROXY}") |
| K8S_DIR: $(yaml-quote "${WINDOWS_K8S_DIR}") |
| NODE_DIR: $(yaml-quote "${WINDOWS_NODE_DIR}") |
| LOGS_DIR: $(yaml-quote "${WINDOWS_LOGS_DIR}") |
| CNI_DIR: $(yaml-quote "${WINDOWS_CNI_DIR}") |
| CNI_CONFIG_DIR: $(yaml-quote "${WINDOWS_CNI_CONFIG_DIR}") |
| WINDOWS_CNI_STORAGE_PATH: $(yaml-quote "${WINDOWS_CNI_STORAGE_PATH}") |
| WINDOWS_CNI_VERSION: $(yaml-quote "${WINDOWS_CNI_VERSION}") |
| WINDOWS_CONTAINER_RUNTIME: $(yaml-quote "${WINDOWS_CONTAINER_RUNTIME}") |
| WINDOWS_CONTAINER_RUNTIME_ENDPOINT: $(yaml-quote "${WINDOWS_CONTAINER_RUNTIME_ENDPOINT:-}") |
| MANIFESTS_DIR: $(yaml-quote "${WINDOWS_MANIFESTS_DIR}") |
| PKI_DIR: $(yaml-quote "${WINDOWS_PKI_DIR}") |
| CA_FILE_PATH: $(yaml-quote "${WINDOWS_CA_FILE}") |
| KUBELET_CONFIG_FILE: $(yaml-quote "${WINDOWS_KUBELET_CONFIG_FILE}") |
| KUBEPROXY_ARGS: $(yaml-quote "${KUBEPROXY_ARGS}") |
| KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_KUBECONFIG_FILE}") |
| BOOTSTRAP_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_BOOTSTRAP_KUBECONFIG_FILE}") |
| KUBEPROXY_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_KUBEPROXY_KUBECONFIG_FILE}") |
| WINDOWS_INFRA_CONTAINER: $(yaml-quote "${WINDOWS_INFRA_CONTAINER}") |
| WINDOWS_ENABLE_PIGZ: $(yaml-quote "${WINDOWS_ENABLE_PIGZ}") |
| WINDOWS_ENABLE_HYPERV: $(yaml-quote "${WINDOWS_ENABLE_HYPERV}") |
| ENABLE_AUTH_PROVIDER_GCP: $(yaml-quote "${ENABLE_AUTH_PROVIDER_GCP}") |
| ENABLE_NODE_PROBLEM_DETECTOR: $(yaml-quote "${WINDOWS_ENABLE_NODE_PROBLEM_DETECTOR}") |
| NODE_PROBLEM_DETECTOR_VERSION: $(yaml-quote "${NODE_PROBLEM_DETECTOR_VERSION}") |
| NODE_PROBLEM_DETECTOR_TAR_HASH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TAR_HASH}") |
| NODE_PROBLEM_DETECTOR_RELEASE_PATH: $(yaml-quote "${NODE_PROBLEM_DETECTOR_RELEASE_PATH}") |
| NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS: $(yaml-quote "${WINDOWS_NODE_PROBLEM_DETECTOR_CUSTOM_FLAGS}") |
| NODE_PROBLEM_DETECTOR_TOKEN: $(yaml-quote "${NODE_PROBLEM_DETECTOR_TOKEN:-}") |
| WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE: $(yaml-quote "${WINDOWS_NODEPROBLEMDETECTOR_KUBECONFIG_FILE}") |
| AUTH_PROVIDER_GCP_STORAGE_PATH: $(yaml-quote "${AUTH_PROVIDER_GCP_STORAGE_PATH}") |
| AUTH_PROVIDER_GCP_VERSION: $(yaml-quote "${AUTH_PROVIDER_GCP_VERSION}") |
| AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64: $(yaml-quote "${AUTH_PROVIDER_GCP_HASH_WINDOWS_AMD64}") |
| AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR: $(yaml-quote "${AUTH_PROVIDER_GCP_WINDOWS_BIN_DIR}") |
| AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE: $(yaml-quote "${AUTH_PROVIDER_GCP_WINDOWS_CONF_FILE}") |
| EOF |
| } |
| |
| function sha512sum-file() { |
| local shasum |
| if which sha512sum >/dev/null 2>&1; then |
| shasum=$(sha512sum "$1") |
| else |
| shasum=$(shasum -a512 "$1") |
| fi |
| echo "${shasum%%[[:blank:]]*}" |
| } |
| |
| # Create certificate pairs for the cluster. |
| # $1: The public IP for the master. |
| # |
| # These are used for static cert distribution (e.g. static clustering) at |
| # cluster creation time. This will be obsoleted once we implement dynamic |
| # clustering. |
| # |
| # The following certificate pairs are created: |
| # |
| # - ca (the cluster's certificate authority) |
| # - server |
| # - kubelet |
| # - kubecfg (for kubectl) |
| # |
| # TODO(roberthbailey): Replace easyrsa with a simple Go program to generate |
| # the certs that we need. |
| # |
| # Assumed vars |
| # KUBE_TEMP |
| # MASTER_NAME |
| # |
| # Vars set: |
| # CERT_DIR |
| # CA_CERT_BASE64 |
| # MASTER_CERT_BASE64 |
| # MASTER_KEY_BASE64 |
| # KUBELET_CERT_BASE64 |
| # KUBELET_KEY_BASE64 |
| # KUBECFG_CERT_BASE64 |
| # KUBECFG_KEY_BASE64 |
| function create-certs { |
| local -r primary_cn="${1}" |
| |
| # Determine extra certificate names for master |
| |
| # Create service_ip by stripping the network mask part from |
| # SERVICE_CLUSTER_IP_RANGE and incrementing the host part with 1 |
| service_ip=${SERVICE_CLUSTER_IP_RANGE%/*} |
| service_ip="${service_ip%.*}.$((${service_ip##*.} + 1))" |
| local sans="" |
| for extra in "$@"; do |
| if [[ -n "${extra}" ]]; then |
| sans="${sans}IP:${extra}," |
| fi |
| done |
| sans="${sans}IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${DNS_DOMAIN},DNS:${MASTER_NAME}" |
| |
| echo "Generating certs for alternate-names: ${sans}" |
| |
| setup-easyrsa |
| PRIMARY_CN="${primary_cn}" SANS="${sans}" generate-certs |
| AGGREGATOR_PRIMARY_CN="${primary_cn}" AGGREGATOR_SANS="${sans}" generate-aggregator-certs |
| KONNECTIVITY_SERVER_PRIMARY_CN="${primary_cn}" KONNECTIVITY_SERVER_SANS="${sans}" generate-konnectivity-server-certs |
| CLOUD_PVL_ADMISSION_PRIMARY_CN="${primary_cn}" CLOUD_PVL_ADMISSION_SANS="${sans}" generate-cloud-pvl-admission-certs |
| |
| # By default, linux wraps base64 output every 76 cols, so we use 'tr -d' to remove whitespaces. |
| # Note 'base64 -w0' doesn't work on Mac OS X, which has different flags. |
| CA_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/ca.key" | tr -d '\r\n') |
| CA_CERT_BASE64=$(base64 "${CERT_DIR}/pki/ca.crt" | tr -d '\r\n') |
| MASTER_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/${MASTER_NAME}.crt" | tr -d '\r\n') |
| MASTER_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/${MASTER_NAME}.key" | tr -d '\r\n') |
| KUBELET_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/kubelet.crt" | tr -d '\r\n') |
| KUBELET_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/kubelet.key" | tr -d '\r\n') |
| KUBECFG_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/kubecfg.crt" | tr -d '\r\n') |
| KUBECFG_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/kubecfg.key" | tr -d '\r\n') |
| KUBEAPISERVER_CERT_BASE64=$(base64 "${CERT_DIR}/pki/issued/kube-apiserver.crt" | tr -d '\r\n') |
| KUBEAPISERVER_KEY_BASE64=$(base64 "${CERT_DIR}/pki/private/kube-apiserver.key" | tr -d '\r\n') |
| |
| # Setting up an addition directory (beyond pki) as it is the simplest way to |
| # ensure we get a different CA pair to sign the proxy-client certs and which |
| # we can send CA public key to the user-apiserver to validate communication. |
| AGGREGATOR_CA_KEY_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n') |
| REQUESTHEADER_CA_CERT_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/ca.crt" | tr -d '\r\n') |
| PROXY_CLIENT_CERT_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/issued/proxy-client.crt" | tr -d '\r\n') |
| PROXY_CLIENT_KEY_BASE64=$(base64 "${AGGREGATOR_CERT_DIR}/pki/private/proxy-client.key" | tr -d '\r\n') |
| |
| # Setting up the Kubernetes API Server Konnectivity Server auth. |
| # This includes certs for both API Server to Konnectivity Server and |
| # Konnectivity Agent to Konnectivity Server. |
| KONNECTIVITY_SERVER_CA_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n') |
| KONNECTIVITY_SERVER_CA_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" | tr -d '\r\n') |
| KONNECTIVITY_SERVER_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n') |
| KONNECTIVITY_SERVER_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" | tr -d '\r\n') |
| KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" | tr -d '\r\n') |
| KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(base64 "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" | tr -d '\r\n') |
| KONNECTIVITY_AGENT_CA_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n') |
| KONNECTIVITY_AGENT_CA_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" | tr -d '\r\n') |
| KONNECTIVITY_AGENT_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n') |
| KONNECTIVITY_AGENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" | tr -d '\r\n') |
| KONNECTIVITY_AGENT_CLIENT_CERT_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" | tr -d '\r\n') |
| KONNECTIVITY_AGENT_CLIENT_KEY_BASE64=$(base64 "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key" | tr -d '\r\n') |
| |
| CLOUD_PVL_ADMISSION_CA_KEY_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/ca.key" | tr -d '\r\n') |
| CLOUD_PVL_ADMISSION_CA_CERT_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/ca.crt" | tr -d '\r\n') |
| CLOUD_PVL_ADMISSION_CERT_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/server.crt" | tr -d '\r\n') |
| CLOUD_PVL_ADMISSION_KEY_BASE64=$(base64 "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/server.key" | tr -d '\r\n') |
| } |
| |
| # Set up easy-rsa directory structure. |
| # |
| # Assumed vars |
| # KUBE_TEMP |
| # |
| # Vars set: |
| # CERT_DIR |
| # AGGREGATOR_CERT_DIR |
| function setup-easyrsa { |
| local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX") |
| # Note: This was heavily cribbed from make-ca-cert.sh |
| (set -x |
| cd "${KUBE_TEMP}" |
| curl -L -O --connect-timeout 20 --retry 6 --retry-delay 2 https://dl.k8s.io/easy-rsa/easy-rsa.tar.gz |
| tar xzf easy-rsa.tar.gz |
| mkdir easy-rsa-master/kubelet |
| cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/kubelet |
| mkdir easy-rsa-master/aggregator |
| cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/aggregator |
| mkdir easy-rsa-master/cloud-pvl-admission |
| cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/cloud-pvl-admission |
| mkdir easy-rsa-master/konnectivity-server |
| cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-server |
| mkdir easy-rsa-master/konnectivity-agent |
| cp -r easy-rsa-master/easyrsa3/* easy-rsa-master/konnectivity-agent) &>"${cert_create_debug_output}" || true |
| CERT_DIR="${KUBE_TEMP}/easy-rsa-master/easyrsa3" |
| AGGREGATOR_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/aggregator" |
| CLOUD_PVL_ADMISSION_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission" |
| KONNECTIVITY_SERVER_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-server" |
| KONNECTIVITY_AGENT_CERT_DIR="${KUBE_TEMP}/easy-rsa-master/konnectivity-agent" |
| if [ ! -x "${CERT_DIR}/easyrsa" ] || [ ! -x "${AGGREGATOR_CERT_DIR}/easyrsa" ]; then |
| # TODO(roberthbailey,porridge): add better error handling here, |
| # see https://github.com/kubernetes/kubernetes/issues/55229 |
| cat "${cert_create_debug_output}" >&2 |
| echo "=== Failed to setup easy-rsa: Aborting ===" >&2 |
| exit 2 |
| fi |
| } |
| |
| # Runs the easy RSA commands to generate certificate files. |
| # The generated files are IN ${CERT_DIR} |
| # |
| # Assumed vars (see shellcheck disable directives below) |
| # KUBE_TEMP |
| # MASTER_NAME |
| # CERT_DIR |
| # PRIMARY_CN: Primary canonical name |
| # SANS: Subject alternate names |
| # |
| # |
| function generate-certs { |
| local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX") |
| # Note: This was heavily cribbed from make-ca-cert.sh |
| (set -x |
| cd "${CERT_DIR}" |
| ./easyrsa init-pki |
| # this puts the cert into pki/ca.crt and the key into pki/private/ca.key |
| # PRIMARY_CN (expected to be) defined by caller |
| # shellcheck disable=SC2153 |
| ./easyrsa --batch "--req-cn=${PRIMARY_CN}@$(date +%s)" build-ca nopass |
| # SANS (expected to be) defined by caller |
| # shellcheck disable=SC2153 |
| ./easyrsa --subject-alt-name="${SANS}" build-server-full "${MASTER_NAME}" nopass |
| ./easyrsa build-client-full kube-apiserver nopass |
| |
| kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" |
| |
| # make the config for the signer |
| echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json" |
| # create the kubelet client cert with the correct groups |
| echo '{"CN":"kubelet","names":[{"O":"system:nodes"}],"hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare kubelet |
| mv "kubelet-key.pem" "pki/private/kubelet.key" |
| mv "kubelet.pem" "pki/issued/kubelet.crt" |
| rm -f "kubelet.csr" |
| |
| # Make a superuser client cert with subject "O=system:masters, CN=kubecfg" |
| ./easyrsa --dn-mode=org \ |
| --req-cn=kubecfg --req-org=system:masters \ |
| --req-c= --req-st= --req-city= --req-email= --req-ou= \ |
| build-client-full kubecfg nopass) &>"${cert_create_debug_output}" || true |
| local output_file_missing=0 |
| local output_file |
| for output_file in \ |
| "${CERT_DIR}/pki/private/ca.key" \ |
| "${CERT_DIR}/pki/ca.crt" \ |
| "${CERT_DIR}/pki/issued/${MASTER_NAME}.crt" \ |
| "${CERT_DIR}/pki/private/${MASTER_NAME}.key" \ |
| "${CERT_DIR}/pki/issued/kubelet.crt" \ |
| "${CERT_DIR}/pki/private/kubelet.key" \ |
| "${CERT_DIR}/pki/issued/kubecfg.crt" \ |
| "${CERT_DIR}/pki/private/kubecfg.key" \ |
| "${CERT_DIR}/pki/issued/kube-apiserver.crt" \ |
| "${CERT_DIR}/pki/private/kube-apiserver.key" |
| do |
| if [[ ! -s "${output_file}" ]]; then |
| echo "Expected file ${output_file} not created" >&2 |
| output_file_missing=1 |
| fi |
| done |
| if [ $output_file_missing -ne 0 ]; then |
| # TODO(roberthbailey,porridge): add better error handling here, |
| # see https://github.com/kubernetes/kubernetes/issues/55229 |
| cat "${cert_create_debug_output}" >&2 |
| echo "=== Failed to generate master certificates: Aborting ===" >&2 |
| exit 2 |
| fi |
| } |
| |
| # Runs the easy RSA commands to generate aggregator certificate files. |
| # The generated files are in ${AGGREGATOR_CERT_DIR} |
| # |
| # Assumed vars |
| # KUBE_TEMP |
| # AGGREGATOR_MASTER_NAME |
| # AGGREGATOR_CERT_DIR |
| # AGGREGATOR_PRIMARY_CN: Primary canonical name |
| # AGGREGATOR_SANS: Subject alternate names |
| # |
| # |
| function generate-aggregator-certs { |
| local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX") |
| # Note: This was heavily cribbed from make-ca-cert.sh |
| (set -x |
| cd "${KUBE_TEMP}/easy-rsa-master/aggregator" |
| ./easyrsa init-pki |
| # this puts the cert into pki/ca.crt and the key into pki/private/ca.key |
| ./easyrsa --batch "--req-cn=${AGGREGATOR_PRIMARY_CN}@$(date +%s)" build-ca nopass |
| ./easyrsa --subject-alt-name="${AGGREGATOR_SANS}" build-server-full "${AGGREGATOR_MASTER_NAME}" nopass |
| ./easyrsa build-client-full aggregator-apiserver nopass |
| |
| kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" |
| |
| # make the config for the signer |
| echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json" |
| # create the aggregator client cert with the correct groups |
| echo '{"CN":"aggregator","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare proxy-client |
| mv "proxy-client-key.pem" "pki/private/proxy-client.key" |
| mv "proxy-client.pem" "pki/issued/proxy-client.crt" |
| rm -f "proxy-client.csr" |
| |
| # Make a superuser client cert with subject "O=system:masters, CN=kubecfg" |
| ./easyrsa --dn-mode=org \ |
| --req-cn=proxy-clientcfg --req-org=system:aggregator \ |
| --req-c= --req-st= --req-city= --req-email= --req-ou= \ |
| build-client-full proxy-clientcfg nopass) &>"${cert_create_debug_output}" || true |
| local output_file_missing=0 |
| local output_file |
| for output_file in \ |
| "${AGGREGATOR_CERT_DIR}/pki/private/ca.key" \ |
| "${AGGREGATOR_CERT_DIR}/pki/ca.crt" \ |
| "${AGGREGATOR_CERT_DIR}/pki/issued/proxy-client.crt" \ |
| "${AGGREGATOR_CERT_DIR}/pki/private/proxy-client.key" |
| do |
| if [[ ! -s "${output_file}" ]]; then |
| echo "Expected file ${output_file} not created" >&2 |
| output_file_missing=1 |
| fi |
| done |
| if [ $output_file_missing -ne 0 ]; then |
| # TODO(roberthbailey,porridge): add better error handling here, |
| # see https://github.com/kubernetes/kubernetes/issues/55229 |
| cat "${cert_create_debug_output}" >&2 |
| echo "=== Failed to generate aggregator certificates: Aborting ===" >&2 |
| exit 2 |
| fi |
| } |
| |
| # Runs the easy RSA commands to generate server side certificate files |
| # for the konnectivity server. This includes both server side to both |
| # konnectivity-server and konnectivity-agent. |
| # The generated files are in ${KONNECTIVITY_SERVER_CERT_DIR} and |
| # ${KONNECTIVITY_AGENT_CERT_DIR} |
| # |
| # Assumed vars |
| # KUBE_TEMP |
| # KONNECTIVITY_SERVER_CERT_DIR |
| # KONNECTIVITY_SERVER_PRIMARY_CN: Primary canonical name |
| # KONNECTIVITY_SERVER_SANS: Subject alternate names |
| # |
| function generate-konnectivity-server-certs { |
| local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX") |
| # Note: This was heavily cribbed from make-ca-cert.sh |
| (set -x |
| # Make the client <-> konnectivity server side certificates. |
| cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-server" |
| ./easyrsa init-pki |
| # this puts the cert into pki/ca.crt and the key into pki/private/ca.key |
| ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass |
| ./easyrsa --subject-alt-name="IP:127.0.0.1,${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass |
| ./easyrsa build-client-full client nopass |
| |
| kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" |
| |
| # make the config for the signer |
| echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json" |
| # create the konnectivity server cert with the correct groups |
| echo '{"CN":"konnectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-server |
| rm -f "konnectivity-server.csr" |
| |
| # Make the agent <-> konnectivity server side certificates. |
| cd "${KUBE_TEMP}/easy-rsa-master/konnectivity-agent" |
| ./easyrsa init-pki |
| # this puts the cert into pki/ca.crt and the key into pki/private/ca.key |
| ./easyrsa --batch "--req-cn=${KONNECTIVITY_SERVER_PRIMARY_CN}@$(date +%s)" build-ca nopass |
| ./easyrsa --subject-alt-name="${KONNECTIVITY_SERVER_SANS}" build-server-full server nopass |
| ./easyrsa build-client-full client nopass |
| |
| kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" |
| |
| # make the config for the signer |
| echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json" |
| # create the konnectivity server cert with the correct groups |
| echo '{"CN":"koonectivity-server","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent |
| rm -f "konnectivity-agent.csr" |
| |
| echo "completed main certificate section") &>"${cert_create_debug_output}" || true |
| |
| local output_file_missing=0 |
| local output_file |
| for output_file in \ |
| "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/ca.key" \ |
| "${KONNECTIVITY_SERVER_CERT_DIR}/pki/ca.crt" \ |
| "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/server.crt" \ |
| "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/server.key" \ |
| "${KONNECTIVITY_SERVER_CERT_DIR}/pki/issued/client.crt" \ |
| "${KONNECTIVITY_SERVER_CERT_DIR}/pki/private/client.key" \ |
| "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/ca.key" \ |
| "${KONNECTIVITY_AGENT_CERT_DIR}/pki/ca.crt" \ |
| "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/server.crt" \ |
| "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/server.key" \ |
| "${KONNECTIVITY_AGENT_CERT_DIR}/pki/issued/client.crt" \ |
| "${KONNECTIVITY_AGENT_CERT_DIR}/pki/private/client.key" |
| do |
| if [[ ! -s "${output_file}" ]]; then |
| echo "Expected file ${output_file} not created" >&2 |
| output_file_missing=1 |
| fi |
| done |
| if (( output_file_missing )); then |
| # TODO(roberthbailey,porridge): add better error handling here, |
| # see https://github.com/kubernetes/kubernetes/issues/55229 |
| cat "${cert_create_debug_output}" >&2 |
| echo "=== Failed to generate konnectivity-server certificates: Aborting ===" >&2 |
| exit 2 |
| fi |
| } |
| |
| # Runs the easy RSA commands to generate server side certificate files |
| # for the cloud-pvl-admission webhook. |
| # The generated files are in ${CLOUD_PVL_ADMISSION_CERT_DIR} |
| # |
| # Assumed vars |
| # KUBE_TEMP |
| # CLOUD_PVL_ADMISSION_CERT_DIR |
| # CLOUD_PVL_ADMISSION_PRIMARY_CN: Primary canonical name |
| # CLOUD_PVL_ADMISSION_SANS: Subject alternate names |
| # |
| function generate-cloud-pvl-admission-certs { |
| local -r cert_create_debug_output=$(mktemp "${KUBE_TEMP}/cert_create_debug_output.XXX") |
| # Note: This was heavily cribbed from make-ca-cert.sh |
| (set -x |
| # Make the client <-> cloud-pvl-admission server side certificates. |
| cd "${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission" |
| ./easyrsa init-pki |
| # this puts the cert into pki/ca.crt and the key into pki/private/ca.key |
| ./easyrsa --batch "--req-cn=${CLOUD_PVL_ADMISSION_PRIMARY_CN}@$(date +%s)" build-ca nopass |
| ./easyrsa --subject-alt-name="IP:127.0.0.1,${CLOUD_PVL_ADMISSION_SANS}" build-server-full server nopass |
| ./easyrsa build-client-full client nopass |
| |
| kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" |
| |
| # make the config for the signer |
| echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","client auth"]}}}' > "ca-config.json" |
| # create the cloud-pvl-admission cert with the correct groups |
| echo '{"CN":"cloud-pvl-admission","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare cloud-pvl-admission |
| rm -f "cloud-pvl-admission.csr" |
| |
| # Make the cloud-pvl-admission server side certificates. |
| cd "${KUBE_TEMP}/easy-rsa-master/cloud-pvl-admission" |
| ./easyrsa init-pki |
| # this puts the cert into pki/ca.crt and the key into pki/private/ca.key |
| ./easyrsa --batch "--req-cn=${CLOUD_PVL_ADMISSION_PRIMARY_CN}@$(date +%s)" build-ca nopass |
| ./easyrsa --subject-alt-name="${CLOUD_PVL_ADMISSION_SANS}" build-server-full server nopass |
| ./easyrsa build-client-full client nopass |
| |
| kube::util::ensure-cfssl "${KUBE_TEMP}/cfssl" |
| |
| # make the config for the signer |
| echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment","agent auth"]}}}' > "ca-config.json" |
| # create the cloud-pvl-admission server cert with the correct groups |
| echo '{"CN":"cloud-pvl-admission","hosts":[""],"key":{"algo":"rsa","size":2048}}' | "${CFSSL_BIN}" gencert -ca=pki/ca.crt -ca-key=pki/private/ca.key -config=ca-config.json - | "${CFSSLJSON_BIN}" -bare konnectivity-agent |
| rm -f "konnectivity-agent.csr" |
| |
| echo "completed main certificate section") &>"${cert_create_debug_output}" || true |
| |
| local output_file_missing=0 |
| local output_file |
| for output_file in \ |
| "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/ca.key" \ |
| "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/ca.crt" \ |
| "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/server.crt" \ |
| "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/server.key" \ |
| "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/issued/client.crt" \ |
| "${CLOUD_PVL_ADMISSION_CERT_DIR}/pki/private/client.key" |
| do |
| if [[ ! -s "${output_file}" ]]; then |
| echo "Expected file ${output_file} not created" >&2 |
| output_file_missing=1 |
| fi |
| done |
| if (( output_file_missing )); then |
| # TODO(roberthbailey,porridge): add better error handling here, |
| # see https://github.com/kubernetes/kubernetes/issues/55229 |
| cat "${cert_create_debug_output}" >&2 |
| echo "=== Failed to generate cloud-pvl-admission certificates: Aborting ===" >&2 |
| exit 2 |
| fi |
| } |
| |
| # Using provided master env, extracts value from provided key. |
| # |
| # Args: |
| # $1 master env (kube-env of master; result of calling get-master-env) |
| # $2 env key to use |
| function get-env-val() { |
| local match |
| match=$( (echo "${1}" | grep -E "^${2}:") || echo '') |
| if [[ -z "${match}" ]]; then |
| echo "" |
| fi |
| echo "${match}" | cut -d : -f 2 | cut -d \' -f 2 |
| } |
| |
| # Load the master env by calling get-master-env, and extract important values |
| function parse-master-env() { |
| # Get required master env vars |
| local master_env |
| master_env=$(get-master-env) |
| KUBE_PROXY_TOKEN=$(get-env-val "${master_env}" "KUBE_PROXY_TOKEN") |
| NODE_PROBLEM_DETECTOR_TOKEN=$(get-env-val "${master_env}" "NODE_PROBLEM_DETECTOR_TOKEN") |
| CA_CERT_BASE64=$(get-env-val "${master_env}" "CA_CERT") |
| CA_KEY_BASE64=$(get-env-val "${master_env}" "CA_KEY") |
| KUBEAPISERVER_CERT_BASE64=$(get-env-val "${master_env}" "KUBEAPISERVER_CERT") |
| KUBEAPISERVER_KEY_BASE64=$(get-env-val "${master_env}" "KUBEAPISERVER_KEY") |
| EXTRA_DOCKER_OPTS=$(get-env-val "${master_env}" "EXTRA_DOCKER_OPTS") |
| KUBELET_CERT_BASE64=$(get-env-val "${master_env}" "KUBELET_CERT") |
| KUBELET_KEY_BASE64=$(get-env-val "${master_env}" "KUBELET_KEY") |
| MASTER_CERT_BASE64=$(get-env-val "${master_env}" "MASTER_CERT") |
| MASTER_KEY_BASE64=$(get-env-val "${master_env}" "MASTER_KEY") |
| AGGREGATOR_CA_KEY_BASE64=$(get-env-val "${master_env}" "AGGREGATOR_CA_KEY") |
| REQUESTHEADER_CA_CERT_BASE64=$(get-env-val "${master_env}" "REQUESTHEADER_CA_CERT") |
| PROXY_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "PROXY_CLIENT_CERT") |
| PROXY_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "PROXY_CLIENT_KEY") |
| ENABLE_LEGACY_ABAC=$(get-env-val "${master_env}" "ENABLE_LEGACY_ABAC") |
| ETCD_APISERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CA_KEY") |
| ETCD_APISERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CA_CERT") |
| ETCD_APISERVER_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_KEY") |
| ETCD_APISERVER_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_SERVER_CERT") |
| ETCD_APISERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_KEY") |
| ETCD_APISERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "ETCD_APISERVER_CLIENT_CERT") |
| CLOUD_PVL_ADMISSION_CA_KEY_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CA_KEY") |
| CLOUD_PVL_ADMISSION_CA_CERT_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CA_CERT") |
| CLOUD_PVL_ADMISSION_CERT_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_CERT") |
| CLOUD_PVL_ADMISSION_KEY_BASE64=$(get-env-val "${master_env}" "CLOUD_PVL_ADMISSION_KEY") |
| KONNECTIVITY_SERVER_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_KEY") |
| KONNECTIVITY_SERVER_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CA_CERT") |
| KONNECTIVITY_SERVER_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CERT") |
| KONNECTIVITY_SERVER_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_KEY") |
| KONNECTIVITY_SERVER_CLIENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_CERT") |
| KONNECTIVITY_SERVER_CLIENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_SERVER_CLIENT_KEY") |
| KONNECTIVITY_AGENT_CA_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_KEY") |
| KONNECTIVITY_AGENT_CA_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CA_CERT") |
| KONNECTIVITY_AGENT_CERT_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_CERT") |
| KONNECTIVITY_AGENT_KEY_BASE64=$(get-env-val "${master_env}" "KONNECTIVITY_AGENT_KEY") |
| } |
| |
| # Update or verify required gcloud components are installed |
| # at minimum required version. |
| # Assumed vars |
| # KUBE_PROMPT_FOR_UPDATE |
| function update-or-verify-gcloud() { |
| local sudo_prefix="" |
| if [ ! -w "$(dirname "$(which gcloud)")" ]; then |
| sudo_prefix="sudo" |
| fi |
| # update and install components as needed |
| # (deliberately word split $gcloud_prompt) |
| # shellcheck disable=SC2086 |
| if [[ "${KUBE_PROMPT_FOR_UPDATE}" == "y" ]]; then |
| ${sudo_prefix} gcloud ${gcloud_prompt:-} components install alpha |
| ${sudo_prefix} gcloud ${gcloud_prompt:-} components install beta |
| ${sudo_prefix} gcloud ${gcloud_prompt:-} components update |
| else |
| local version |
| version=$(gcloud version --format=json) |
| python3 -c" |
| import json,sys |
| from distutils import version |
| |
| minVersion = version.LooseVersion('1.3.0') |
| required = [ 'alpha', 'beta', 'core' ] |
| data = json.loads(sys.argv[1]) |
| rel = data.get('Google Cloud SDK') |
| if 'CL @' in rel: |
| print('Using dev version of gcloud: %s' %rel) |
| exit(0) |
| if rel != 'HEAD' and version.LooseVersion(rel) < minVersion: |
| print('gcloud version out of date ( < %s )' % minVersion) |
| exit(1) |
| missing = [] |
| for c in required: |
| if not data.get(c): |
| missing += [c] |
| if missing: |
| for c in missing: |
| print ('missing required gcloud component \"{0}\"'.format(c)) |
| print ('Try running \$(gcloud components install {0})'.format(c)) |
| exit(1) |
| " "${version}" |
| fi |
| } |
| |
| # Robustly try to create a static ip. |
| # $1: The name of the ip to create |
| # $2: The name of the region to create the ip in. |
| function create-static-ip() { |
| detect-project |
| local attempt=0 |
| local REGION="$2" |
| while true; do |
| if gcloud compute addresses create "$1" \ |
| --project "${PROJECT}" \ |
| --region "${REGION}" -q > /dev/null; then |
| # successful operation - wait until it's visible |
| start="$(date +%s)" |
| while true; do |
| now="$(date +%s)" |
| # Timeout set to 15 minutes |
| if [[ $((now - start)) -gt 900 ]]; then |
| echo "Timeout while waiting for master IP visibility" |
| exit 2 |
| fi |
| if gcloud compute addresses describe "$1" --project "${PROJECT}" --region "${REGION}" >/dev/null 2>&1; then |
| break |
| fi |
| echo "Master IP not visible yet. Waiting..." |
| sleep 5 |
| done |
| break |
| fi |
| |
| if gcloud compute addresses describe "$1" \ |
| --project "${PROJECT}" \ |
| --region "${REGION}" >/dev/null 2>&1; then |
| # it exists - postcondition satisfied |
| break |
| fi |
| |
| if (( attempt > 4 )); then |
| echo -e "${color_red}Failed to create static ip $1 ${color_norm}" >&2 |
| exit 2 |
| fi |
| attempt=$((attempt + 1)) |
| echo -e "${color_yellow:-}Attempt $attempt failed to create static ip $1. Retrying.${color_norm:-}" >&2 |
| sleep $((attempt * 5)) |
| done |
| } |
| |
| # Robustly try to create a firewall rule. |
| # $1: The name of firewall rule. |
| # $2: IP ranges. |
| # $3: Target tags for this firewall rule. |
| function create-firewall-rule() { |
| detect-project |
| local attempt=0 |
| while true; do |
| if ! gcloud compute firewall-rules create "$1" \ |
| --project "${NETWORK_PROJECT}" \ |
| --network "${NETWORK}" \ |
| --source-ranges "$2" \ |
| --target-tags "$3" \ |
| --allow tcp,udp,icmp,esp,ah,sctp; then |
| if (( attempt > 4 )); then |
| echo -e "${color_red}Failed to create firewall rule $1 ${color_norm}" >&2 |
| exit 2 |
| fi |
| echo -e "${color_yellow}Attempt $((attempt + 1)) failed to create firewall rule $1. Retrying.${color_norm}" >&2 |
| attempt=$((attempt + 1)) |
| sleep $((attempt * 5)) |
| else |
| break |
| fi |
| done |
| } |
| |
| # Format the string argument for gcloud network. |
| function make-gcloud-network-argument() { |
| local network_project="$1" |
| local region="$2" |
| local network="$3" |
| local subnet="$4" |
| local address="$5" # optional |
| local enable_ip_alias="$6" # optional |
| local alias_size="$7" # optional |
| |
| local networkURL="projects/${network_project}/global/networks/${network}" |
| local subnetURL="projects/${network_project}/regions/${region}/subnetworks/${subnet:-}" |
| |
| local ret="" |
| |
| if [[ "${enable_ip_alias}" == 'true' ]]; then |
| ret="--network-interface" |
| ret="${ret} network=${networkURL}" |
| if [[ "${address:-}" == "no-address" ]]; then |
| ret="${ret},no-address" |
| else |
| ret="${ret},address=${address:-}" |
| fi |
| ret="${ret},subnet=${subnetURL}" |
| ret="${ret},aliases=pods-default:${alias_size}" |
| ret="${ret} --no-can-ip-forward" |
| else |
| if [[ -n ${subnet:-} ]]; then |
| ret="${ret} --subnet ${subnetURL}" |
| else |
| ret="${ret} --network ${networkURL}" |
| fi |
| |
| ret="${ret} --can-ip-forward" |
| if [[ -n ${address:-} ]] && [[ "$address" != "no-address" ]]; then |
| ret="${ret} --address ${address}" |
| fi |
| fi |
| |
| echo "${ret}" |
| } |
| |
| # $1: version (required) |
| # $2: Prefix for the template name, i.e. NODE_INSTANCE_PREFIX or |
| # WINDOWS_NODE_INSTANCE_PREFIX. |
| function get-template-name-from-version() { |
| local -r version=${1} |
| local -r template_prefix=${2} |
| # trim template name to pass gce name validation |
| echo "${template_prefix}-template-${version}" | cut -c 1-63 | sed 's/[\.\+]/-/g;s/-*$//g' |
| } |
| |
| # validates the NODE_LOCAL_SSDS_EXT variable |
| function validate-node-local-ssds-ext(){ |
| ssdopts="${1}" |
| |
| if [[ -z "${ssdopts[0]}" || -z "${ssdopts[1]}" || -z "${ssdopts[2]}" ]]; then |
| echo -e "${color_red}Local SSD: NODE_LOCAL_SSDS_EXT is malformed, found ${ssdopts[0]-_},${ssdopts[1]-_},${ssdopts[2]-_} ${color_norm}" >&2 |
| exit 2 |
| fi |
| if [[ "${ssdopts[1]}" != "scsi" && "${ssdopts[1]}" != "nvme" ]]; then |
| echo -e "${color_red}Local SSD: Interface must be scsi or nvme, found: ${ssdopts[1]} ${color_norm}" >&2 |
| exit 2 |
| fi |
| if [[ "${ssdopts[2]}" != "fs" && "${ssdopts[2]}" != "block" ]]; then |
| echo -e "${color_red}Local SSD: Filesystem type must be fs or block, found: ${ssdopts[2]} ${color_norm}" >&2 |
| exit 2 |
| fi |
| local_ssd_ext_count=$((local_ssd_ext_count+ssdopts[0])) |
| if [[ "${local_ssd_ext_count}" -gt "${GCE_MAX_LOCAL_SSD}" || "${local_ssd_ext_count}" -lt 1 ]]; then |
| echo -e "${color_red}Local SSD: Total number of local ssds must range from 1 to 8, found: ${local_ssd_ext_count} ${color_norm}" >&2 |
| exit 2 |
| fi |
| } |
| |
| # Robustly try to create an instance template. |
| # $1: The name of the instance template. |
| # $2: The scopes flag. |
| # $3: String of comma-separated metadata-from-file entries. |
| # $4: String of comma-separated metadata (key=value) entries. |
| # $5: the node OS ("linux" or "windows"). |
| function create-node-template() { |
| detect-project |
| detect-subnetworks |
| local template_name="$1" |
| local metadata_values="$4" |
| local os="$5" |
| local machine_type="$6" |
| |
| # First, ensure the template doesn't exist. |
| # TODO(zmerlynn): To make this really robust, we need to parse the output and |
| # add retries. Just relying on a non-zero exit code doesn't |
| # distinguish an ephemeral failed call from a "not-exists". |
| if gcloud compute instance-templates describe "${template_name}" --project "${PROJECT}" &>/dev/null; then |
| echo "Instance template ${1} already exists; deleting." >&2 |
| if ! gcloud compute instance-templates delete "${template_name}" --project "${PROJECT}" --quiet &>/dev/null; then |
| echo -e "${color_yellow}Failed to delete existing instance template${color_norm}" >&2 |
| exit 2 |
| fi |
| fi |
| |
| local gcloud="gcloud" |
| |
| local accelerator_args=() |
| # VMs with Accelerators cannot be live migrated. |
| # More details here - https://cloud.google.com/compute/docs/gpus/add-gpus#create-new-gpu-instance |
| if [[ -n "${NODE_ACCELERATORS}" ]]; then |
| accelerator_args+=(--maintenance-policy TERMINATE --restart-on-failure --accelerator "${NODE_ACCELERATORS}") |
| gcloud="gcloud beta" |
| fi |
| |
| local preemptible_minions=() |
| if [[ "${PREEMPTIBLE_NODE}" == "true" ]]; then |
| preemptible_minions+=(--preemptible --maintenance-policy TERMINATE) |
| fi |
| |
| local local_ssds=() |
| local_ssd_ext_count=0 |
| if [[ -n "${NODE_LOCAL_SSDS_EXT:-}" ]]; then |
| IFS=";" read -r -a ssdgroups <<< "${NODE_LOCAL_SSDS_EXT:-}" |
| for ssdgroup in "${ssdgroups[@]}" |
| do |
| IFS="," read -r -a ssdopts <<< "${ssdgroup}" |
| validate-node-local-ssds-ext "${ssdopts[@]}" |
| for ((i=1; i<=ssdopts[0]; i++)); do |
| local_ssds+=("--local-ssd=interface=${ssdopts[1]}") |
| done |
| done |
| fi |
| |
| if [[ -n ${NODE_LOCAL_SSDS+x} ]]; then |
| # The NODE_LOCAL_SSDS check below fixes issue #49171 |
| for ((i=1; i<=NODE_LOCAL_SSDS; i++)); do |
| local_ssds+=('--local-ssd=interface=SCSI') |
| done |
| fi |
| |
| local address="" |
| if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then |
| address="no-address" |
| fi |
| |
| local network |
| network=$(make-gcloud-network-argument \ |
| "${NETWORK_PROJECT}" \ |
| "${REGION}" \ |
| "${NETWORK}" \ |
| "${SUBNETWORK:-}" \ |
| "${address}" \ |
| "${ENABLE_IP_ALIASES:-}" \ |
| "${IP_ALIAS_SIZE:-}") |
| |
| local node_image_flags=() |
| if [[ "${os}" == 'linux' ]]; then |
| node_image_flags+=(--image-project "${NODE_IMAGE_PROJECT}" --image "${NODE_IMAGE}") |
| elif [[ "${os}" == 'windows' ]]; then |
| node_image_flags+=(--image-project "${WINDOWS_NODE_IMAGE_PROJECT}" --image "${WINDOWS_NODE_IMAGE}") |
| else |
| echo "Unknown OS ${os}" >&2 |
| exit 1 |
| fi |
| |
| local metadata_flag="${metadata_values:+--metadata ${metadata_values}}" |
| |
| local attempt=1 |
| while true; do |
| echo "Attempt ${attempt} to create ${1}" >&2 |
| # Deliberately word split ${network}, $2 and ${metadata_flag} |
| # shellcheck disable=SC2086 |
| if ! ${gcloud} compute instance-templates create \ |
| "${template_name}" \ |
| --project "${PROJECT}" \ |
| --machine-type "${machine_type}" \ |
| --boot-disk-type "${NODE_DISK_TYPE}" \ |
| --boot-disk-size "${NODE_DISK_SIZE}" \ |
| "${node_image_flags[@]}" \ |
| --service-account "${NODE_SERVICE_ACCOUNT}" \ |
| --tags "${NODE_TAG}" \ |
| "${accelerator_args[@]}" \ |
| "${local_ssds[@]}" \ |
| --region "${REGION}" \ |
| ${network} \ |
| "${preemptible_minions[@]}" \ |
| $2 \ |
| --metadata-from-file "$3" \ |
| ${metadata_flag} >&2; then |
| if (( attempt > 5 )); then |
| echo -e "${color_red}Failed to create instance template ${template_name} ${color_norm}" >&2 |
| exit 2 |
| fi |
| echo -e "${color_yellow}Attempt ${attempt} failed to create instance template ${template_name}. Retrying.${color_norm}" >&2 |
| attempt=$((attempt + 1)) |
| sleep $((attempt * 5)) |
| |
| # In case the previous attempt failed with something like a |
| # Backend Error and left the entry laying around, delete it |
| # before we try again. |
| gcloud compute instance-templates delete "${template_name}" --project "${PROJECT}" &>/dev/null || true |
| else |
| break |
| fi |
| done |
| } |
| |
| # Instantiate a kubernetes cluster |
| # |
| # Assumed vars |
| # KUBE_ROOT |
| # <Various vars set in config file> |
| function kube-up() { |
| kube::util::ensure-temp-dir |
| detect-project |
| |
| load-or-gen-kube-basicauth |
| load-or-gen-kube-bearertoken |
| |
| # Make sure we have the tar files staged on Google Storage |
| find-release-tars |
| upload-tars |
| |
| # ensure that environmental variables specifying number of migs to create |
| set_num_migs |
| |
| if [[ ${KUBE_USE_EXISTING_MASTER:-} == "true" ]]; then |
| detect-master |
| parse-master-env |
| create-subnetworks |
| detect-subnetworks |
| # Windows nodes take longer to boot and setup so create them first. |
| create-windows-nodes |
| create-linux-nodes |
| elif [[ ${KUBE_REPLICATE_EXISTING_MASTER:-} == "true" ]]; then |
| detect-master |
| if [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "ubuntu" ]]; then |
| echo "Master replication supported only for gci and ubuntu" |
| return 1 |
| fi |
| if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then |
| create-internal-loadbalancer |
| fi |
| create-loadbalancer |
| # If replication of master fails, we need to ensure that the replica is removed from etcd clusters. |
| if ! replicate-master; then |
| remove-replica-from-etcd 2379 true || true |
| remove-replica-from-etcd 4002 false || true |
| fi |
| else |
| check-existing |
| create-network |
| create-subnetworks |
| detect-subnetworks |
| create-cloud-nat-router |
| write-cluster-location |
| write-cluster-name |
| create-autoscaler-config |
| create-master |
| create-nodes-firewall |
| create-nodes-template |
| if [[ "${KUBE_CREATE_NODES}" == "true" ]]; then |
| # Windows nodes take longer to boot and setup so create them first. |
| create-windows-nodes |
| create-linux-nodes |
| fi |
| check-cluster |
| fi |
| } |
| |
| function check-existing() { |
| local running_in_terminal=false |
| # May be false if tty is not allocated (for example with ssh -T). |
| if [[ -t 1 ]]; then |
| running_in_terminal=true |
| fi |
| |
| if [[ ${running_in_terminal} == "true" || ${KUBE_UP_AUTOMATIC_CLEANUP} == "true" ]]; then |
| if ! check-resources; then |
| local run_kube_down="n" |
| echo "${KUBE_RESOURCE_FOUND} found." >&2 |
| # Get user input only if running in terminal. |
| if [[ ${running_in_terminal} == "true" && ${KUBE_UP_AUTOMATIC_CLEANUP} == "false" ]]; then |
| read -r -p "Would you like to shut down the old cluster (call kube-down)? [y/N] " run_kube_down |
| fi |
| if [[ ${run_kube_down} == "y" || ${run_kube_down} == "Y" || ${KUBE_UP_AUTOMATIC_CLEANUP} == "true" ]]; then |
| echo "... calling kube-down" >&2 |
| kube-down |
| fi |
| fi |
| fi |
| } |
| |
| function check-network-mode() { |
| local mode |
| mode=$(gcloud compute networks list --filter="name=('${NETWORK}')" --project "${NETWORK_PROJECT}" --format='value(x_gcloud_subnet_mode)' || true) |
| # The deprecated field uses lower case. Convert to upper case for consistency. |
| echo "$mode" | tr '[:lower:]' '[:upper:]' |
| } |
| |
| function create-network() { |
| if ! gcloud compute networks --project "${NETWORK_PROJECT}" describe "${NETWORK}" &>/dev/null; then |
| # The network needs to be created synchronously or we have a race. The |
| # firewalls can be added concurrent with instance creation. |
| local network_mode="auto" |
| if [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" ]]; then |
| network_mode="custom" |
| fi |
| echo "Creating new ${network_mode} network: ${NETWORK}" |
| gcloud compute networks create --project "${NETWORK_PROJECT}" "${NETWORK}" --subnet-mode="${network_mode}" |
| else |
| PREEXISTING_NETWORK=true |
| PREEXISTING_NETWORK_MODE="$(check-network-mode)" |
| echo "Found existing network ${NETWORK} in ${PREEXISTING_NETWORK_MODE} mode." |
| fi |
| |
| if ! gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${CLUSTER_NAME}-default-internal-master" &>/dev/null; then |
| gcloud compute firewall-rules create "${CLUSTER_NAME}-default-internal-master" \ |
| --project "${NETWORK_PROJECT}" \ |
| --network "${NETWORK}" \ |
| --source-ranges "10.0.0.0/8" \ |
| --allow "tcp:1-2379,tcp:2382-65535,udp:1-65535,icmp" \ |
| --target-tags "${MASTER_TAG}"& |
| fi |
| |
| if ! gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${CLUSTER_NAME}-default-internal-node" &>/dev/null; then |
| gcloud compute firewall-rules create "${CLUSTER_NAME}-default-internal-node" \ |
| --project "${NETWORK_PROJECT}" \ |
| --network "${NETWORK}" \ |
| --source-ranges "10.0.0.0/8" \ |
| --allow "tcp:1-65535,udp:1-65535,icmp" \ |
| --target-tags "${NODE_TAG}"& |
| fi |
| |
| if ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NETWORK}-default-ssh" &>/dev/null; then |
| gcloud compute firewall-rules create "${NETWORK}-default-ssh" \ |
| --project "${NETWORK_PROJECT}" \ |
| --network "${NETWORK}" \ |
| --source-ranges "0.0.0.0/0" \ |
| --allow "tcp:22" & |
| fi |
| |
| # Open up TCP 3389 to allow RDP connections. |
| if [[ ${NUM_WINDOWS_NODES} -gt 0 ]]; then |
| if ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NETWORK}-default-rdp" &>/dev/null; then |
| gcloud compute firewall-rules create "${NETWORK}-default-rdp" \ |
| --project "${NETWORK_PROJECT}" \ |
| --network "${NETWORK}" \ |
| --source-ranges "0.0.0.0/0" \ |
| --allow "tcp:3389" & |
| fi |
| fi |
| |
| kube::util::wait-for-jobs || { |
| code=$? |
| echo -e "${color_red}Failed to create firewall rules.${color_norm}" >&2 |
| exit $code |
| } |
| } |
| |
| function expand-default-subnetwork() { |
| gcloud compute networks update "${NETWORK}" \ |
| --switch-to-custom-subnet-mode \ |
| --project "${NETWORK_PROJECT}" \ |
| --quiet || true |
| gcloud compute networks subnets expand-ip-range "${NETWORK}" \ |
| --region="${REGION}" \ |
| --project "${NETWORK_PROJECT}" \ |
| --prefix-length=19 \ |
| --quiet |
| } |
| |
| function create-subnetworks() { |
| case ${ENABLE_IP_ALIASES} in |
| true) echo "IP aliases are enabled. Creating subnetworks.";; |
| false) |
| echo "IP aliases are disabled." |
| if [[ "${ENABLE_BIG_CLUSTER_SUBNETS}" = "true" ]]; then |
| if [[ "${PREEXISTING_NETWORK}" != "true" ]]; then |
| expand-default-subnetwork |
| else |
| echo "${color_yellow}Using pre-existing network ${NETWORK}, subnets won't be expanded to /19!${color_norm}" |
| fi |
| elif [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" && "${PREEXISTING_NETWORK}" != "true" ]]; then |
| gcloud compute networks subnets create "${SUBNETWORK}" --project "${NETWORK_PROJECT}" --region "${REGION}" --network "${NETWORK}" --range "${NODE_IP_RANGE}" |
| fi |
| return;; |
| *) echo "${color_red}Invalid argument to ENABLE_IP_ALIASES${color_norm}" |
| exit 1;; |
| esac |
| |
| # Look for the alias subnet, it must exist and have a secondary |
| # range configured. |
| local subnet |
| subnet=$(gcloud compute networks subnets describe \ |
| --project "${NETWORK_PROJECT}" \ |
| --region "${REGION}" \ |
| "${IP_ALIAS_SUBNETWORK}" 2>/dev/null || true) |
| if [[ -z "${subnet}" ]]; then |
| echo "Creating subnet ${NETWORK}:${IP_ALIAS_SUBNETWORK}" |
| gcloud compute networks subnets create \ |
| "${IP_ALIAS_SUBNETWORK}" \ |
| --description "Automatically generated subnet for ${INSTANCE_PREFIX} cluster. This will be removed on cluster teardown." \ |
| --project "${NETWORK_PROJECT}" \ |
| --network "${NETWORK}" \ |
| --region "${REGION}" \ |
| --range "${NODE_IP_RANGE}" \ |
| --secondary-range "pods-default=${CLUSTER_IP_RANGE}" \ |
| --secondary-range "services-default=${SERVICE_CLUSTER_IP_RANGE}" |
| echo "Created subnetwork ${IP_ALIAS_SUBNETWORK}" |
| else |
| if ! echo "${subnet}" | grep --quiet secondaryIpRanges; then |
| echo "${color_red}Subnet ${IP_ALIAS_SUBNETWORK} does not have a secondary range${color_norm}" |
| exit 1 |
| fi |
| fi |
| } |
| |
| # detect-subnetworks sets the SUBNETWORK var if not already set |
| # Assumed vars: |
| # NETWORK |
| # REGION |
| # NETWORK_PROJECT |
| # |
| # Optional vars: |
| # SUBNETWORK |
| # IP_ALIAS_SUBNETWORK |
| function detect-subnetworks() { |
| if [[ -n ${SUBNETWORK:-} ]]; then |
| echo "Using subnet ${SUBNETWORK}" |
| return 0 |
| fi |
| |
| if [[ -n ${IP_ALIAS_SUBNETWORK:-} ]]; then |
| SUBNETWORK=${IP_ALIAS_SUBNETWORK} |
| echo "Using IP Alias subnet ${SUBNETWORK}" |
| return 0 |
| fi |
| |
| SUBNETWORK=$(gcloud compute networks subnets list \ |
| --network="${NETWORK}" \ |
| --regions="${REGION}" \ |
| --project="${NETWORK_PROJECT}" \ |
| --limit=1 \ |
| --format='value(name)' 2>/dev/null) |
| |
| if [[ -n ${SUBNETWORK:-} ]]; then |
| echo "Found subnet for region ${REGION} in network ${NETWORK}: ${SUBNETWORK}" |
| return 0 |
| fi |
| |
| echo "${color_red}Could not find subnetwork with region ${REGION}, network ${NETWORK}, and project ${NETWORK_PROJECT}" |
| } |
| |
| # Sets up Cloud NAT for the network. |
| # Assumed vars: |
| # NETWORK_PROJECT |
| # REGION |
| # NETWORK |
| function create-cloud-nat-router() { |
| if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then |
| if gcloud compute routers describe "$NETWORK-nat-router" --project "$NETWORK_PROJECT" --region "$REGION" &>/dev/null; then |
| echo "Cloud nat already exists" |
| return 0 |
| fi |
| gcloud compute routers create "$NETWORK-nat-router" \ |
| --project "$NETWORK_PROJECT" \ |
| --region "$REGION" \ |
| --network "$NETWORK" |
| gcloud compute routers nats create "$NETWORK-nat-config" \ |
| --project "$NETWORK_PROJECT" \ |
| --router-region "$REGION" \ |
| --router "$NETWORK-nat-router" \ |
| --nat-primary-subnet-ip-ranges \ |
| --auto-allocate-nat-external-ips \ |
| ${GCE_PRIVATE_CLUSTER_PORTS_PER_VM:+--min-ports-per-vm ${GCE_PRIVATE_CLUSTER_PORTS_PER_VM}} |
| fi |
| } |
| |
| function delete-all-firewall-rules() { |
| local -a fws |
| kube::util::read-array fws < <(gcloud compute firewall-rules list --project "${NETWORK_PROJECT}" --filter="network=${NETWORK}" --format="value(name)") |
| if (( "${#fws[@]}" > 0 )); then |
| echo "Deleting firewall rules remaining in network ${NETWORK}: ${fws[*]}" |
| delete-firewall-rules "${fws[@]}" |
| else |
| echo "No firewall rules in network ${NETWORK}" |
| fi |
| } |
| |
| # Ignores firewall rule arguments that do not exist in NETWORK_PROJECT. |
| function delete-firewall-rules() { |
| for fw in "$@"; do |
| if [[ -n $(gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then |
| gcloud compute firewall-rules delete --project "${NETWORK_PROJECT}" --quiet "${fw}" & |
| fi |
| done |
| kube::util::wait-for-jobs || { |
| echo -e "${color_red}Failed to delete firewall rules.${color_norm}" >&2 |
| } |
| } |
| |
| function delete-network() { |
| if [[ -n $(gcloud compute networks --project "${NETWORK_PROJECT}" describe "${NETWORK}" --format='value(name)' 2>/dev/null || true) ]]; then |
| if ! gcloud compute networks delete --project "${NETWORK_PROJECT}" --quiet "${NETWORK}"; then |
| echo "Failed to delete network '${NETWORK}'. Listing firewall-rules:" |
| gcloud compute firewall-rules --project "${NETWORK_PROJECT}" list --filter="network=${NETWORK}" |
| return 1 |
| fi |
| fi |
| } |
| |
| function delete-cloud-nat-router() { |
| if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then |
| if [[ -n $(gcloud compute routers describe --project "${NETWORK_PROJECT}" --region "${REGION}" "${NETWORK}-nat-router" --format='value(name)' 2>/dev/null || true) ]]; then |
| echo "Deleting Cloud NAT router..." |
| gcloud compute routers delete --project "${NETWORK_PROJECT}" --region "${REGION}" --quiet "${NETWORK}-nat-router" |
| fi |
| fi |
| } |
| |
| function delete-subnetworks() { |
| # If running in custom mode network we need to delete subnets manually. |
| mode="$(check-network-mode)" |
| if [[ "${mode}" == "CUSTOM" ]]; then |
| if [[ "${ENABLE_BIG_CLUSTER_SUBNETS}" = "true" ]]; then |
| echo "Deleting default subnets..." |
| # This value should be kept in sync with number of regions. |
| local parallelism=9 |
| gcloud compute networks subnets list --network="${NETWORK}" --project "${NETWORK_PROJECT}" --format='value(region.basename())' | \ |
| xargs -I {} -P ${parallelism} gcloud --quiet compute networks subnets delete "${NETWORK}" --project "${NETWORK_PROJECT}" --region="{}" || true |
| elif [[ "${CREATE_CUSTOM_NETWORK:-}" == "true" ]]; then |
| echo "Deleting custom subnet..." |
| gcloud --quiet compute networks subnets delete "${SUBNETWORK}" --project "${NETWORK_PROJECT}" --region="${REGION}" || true |
| fi |
| return |
| fi |
| |
| # If we reached here, it means we're not using custom network. |
| # So the only thing we need to check is if IP-aliases was turned |
| # on and we created a subnet for it. If so, we should delete it. |
| if [[ ${ENABLE_IP_ALIASES:-} == "true" ]]; then |
| # Only delete the subnet if we created it (i.e it's not pre-existing). |
| if [[ -z "${KUBE_GCE_IP_ALIAS_SUBNETWORK:-}" ]]; then |
| echo "Removing auto-created subnet ${NETWORK}:${IP_ALIAS_SUBNETWORK}" |
| if [[ -n $(gcloud compute networks subnets describe \ |
| --project "${NETWORK_PROJECT}" \ |
| --region "${REGION}" \ |
| "${IP_ALIAS_SUBNETWORK}" 2>/dev/null) ]]; then |
| gcloud --quiet compute networks subnets delete \ |
| --project "${NETWORK_PROJECT}" \ |
| --region "${REGION}" \ |
| "${IP_ALIAS_SUBNETWORK}" |
| fi |
| fi |
| fi |
| } |
| |
| # Generates SSL certificates for etcd cluster peer to peer communication. Uses cfssl program. |
| # |
| # Assumed vars: |
| # KUBE_TEMP: temporary directory |
| # |
| # Args: |
| # $1: host name |
| # $2: CA certificate |
| # $3: CA key |
| # |
| # If CA cert/key is empty, the function will also generate certs for CA. |
| # |
| # Vars set: |
| # ETCD_CA_KEY_BASE64 |
| # ETCD_CA_CERT_BASE64 |
| # ETCD_PEER_KEY_BASE64 |
| # ETCD_PEER_CERT_BASE64 |
| # |
| function create-etcd-certs { |
| local host=${1} |
| local ca_cert=${2:-} |
| local ca_key=${3:-} |
| |
| GEN_ETCD_CA_CERT="${ca_cert}" GEN_ETCD_CA_KEY="${ca_key}" \ |
| generate-etcd-cert "${KUBE_TEMP}/cfssl" "${host}" "peer" "peer" |
| |
| pushd "${KUBE_TEMP}/cfssl" |
| ETCD_CA_KEY_BASE64=$(base64 "ca-key.pem" | tr -d '\r\n') |
| ETCD_CA_CERT_BASE64=$(gzip -c "ca.pem" | base64 | tr -d '\r\n') |
| ETCD_PEER_KEY_BASE64=$(base64 "peer-key.pem" | tr -d '\r\n') |
| ETCD_PEER_CERT_BASE64=$(gzip -c "peer.pem" | base64 | tr -d '\r\n') |
| popd |
| } |
| |
| # Generates SSL certificates for etcd-client and kube-apiserver communication. Uses cfssl program. |
| # |
| # Assumed vars: |
| # KUBE_TEMP: temporary directory |
| # |
| # Args: |
| # $1: host server name |
| # $2: host client name |
| # $3: CA certificate |
| # $4: CA key |
| # |
| # If CA cert/key is empty, the function will also generate certs for CA. |
| # |
| # Vars set: |
| # ETCD_APISERVER_CA_KEY_BASE64 |
| # ETCD_APISERVER_CA_CERT_BASE64 |
| # ETCD_APISERVER_SERVER_KEY_BASE64 |
| # ETCD_APISERVER_SERVER_CERT_BASE64 |
| # ETCD_APISERVER_CLIENT_KEY_BASE64 |
| # ETCD_APISERVER_CLIENT_CERT_BASE64 |
| # |
| function create-etcd-apiserver-certs { |
| local hostServer=${1} |
| local hostClient=${2} |
| local etcd_apiserver_ca_cert=${3:-} |
| local etcd_apiserver_ca_key=${4:-} |
| |
| GEN_ETCD_CA_CERT="${etcd_apiserver_ca_cert}" GEN_ETCD_CA_KEY="${etcd_apiserver_ca_key}" \ |
| generate-etcd-cert "${KUBE_TEMP}/cfssl" "${hostServer}" "server" "etcd-apiserver-server" |
| generate-etcd-cert "${KUBE_TEMP}/cfssl" "${hostClient}" "client" "etcd-apiserver-client" |
| |
| pushd "${KUBE_TEMP}/cfssl" |
| ETCD_APISERVER_CA_KEY_BASE64=$(base64 "ca-key.pem" | tr -d '\r\n') |
| ETCD_APISERVER_CA_CERT_BASE64=$(gzip -c "ca.pem" | base64 | tr -d '\r\n') |
| ETCD_APISERVER_SERVER_KEY_BASE64=$(base64 "etcd-apiserver-server-key.pem" | tr -d '\r\n') |
| ETCD_APISERVER_SERVER_CERT_BASE64=$(gzip -c "etcd-apiserver-server.pem" | base64 | tr -d '\r\n') |
| ETCD_APISERVER_CLIENT_KEY_BASE64=$(base64 "etcd-apiserver-client-key.pem" | tr -d '\r\n') |
| ETCD_APISERVER_CLIENT_CERT_BASE64=$(gzip -c "etcd-apiserver-client.pem" | base64 | tr -d '\r\n') |
| popd |
| } |
| |
| |
| function create-master() { |
| echo "Starting master and configuring firewalls" |
| gcloud compute firewall-rules create "${MASTER_NAME}-https" \ |
| --project "${NETWORK_PROJECT}" \ |
| --network "${NETWORK}" \ |
| --target-tags "${MASTER_TAG}" \ |
| --allow tcp:443 & |
| |
| echo "Configuring firewall for apiserver konnectivity server" |
| if [[ "${PREPARE_KONNECTIVITY_SERVICE:-false}" == "true" ]]; then |
| gcloud compute firewall-rules create "${MASTER_NAME}-konnectivity-server" \ |
| --project "${NETWORK_PROJECT}" \ |
| --network "${NETWORK}" \ |
| --target-tags "${MASTER_TAG}" \ |
| --allow tcp:8132 & |
| fi |
| |
| # We have to make sure the disk is created before creating the master VM, so |
| # run this in the foreground. |
| gcloud compute disks create "${MASTER_NAME}-pd" \ |
| --project "${PROJECT}" \ |
| --zone "${ZONE}" \ |
| --type "${MASTER_DISK_TYPE}" \ |
| --size "${MASTER_DISK_SIZE}" |
| |
| # Create rule for accessing and securing etcd servers. |
| if ! gcloud compute firewall-rules --project "${NETWORK_PROJECT}" describe "${MASTER_NAME}-etcd" &>/dev/null; then |
| gcloud compute firewall-rules create "${MASTER_NAME}-etcd" \ |
| --project "${NETWORK_PROJECT}" \ |
| --network "${NETWORK}" \ |
| --source-tags "${MASTER_TAG}" \ |
| --allow "tcp:2380,tcp:2381" \ |
| --target-tags "${MASTER_TAG}" & |
| fi |
| |
| # Generate a bearer token for this cluster. We push this separately |
| # from the other cluster variables so that the client (this |
| # computer) can forget it later. This should disappear with |
| # http://issue.k8s.io/3168 |
| KUBE_PROXY_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) |
| if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then |
| NODE_PROBLEM_DETECTOR_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null) |
| fi |
| |
| # Reserve the master's IP so that it can later be transferred to another VM |
| # without disrupting the kubelets. |
| create-static-ip "${MASTER_NAME}-ip" "${REGION}" |
| MASTER_RESERVED_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \ |
| --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') |
| |
| if [[ "${REGISTER_MASTER_KUBELET:-}" == "true" ]]; then |
| KUBELET_APISERVER="${MASTER_RESERVED_IP}" |
| fi |
| |
| KUBERNETES_MASTER_NAME="${MASTER_RESERVED_IP}" |
| MASTER_ADVERTISE_ADDRESS="${MASTER_RESERVED_IP}" |
| |
| MASTER_INTERNAL_IP="" |
| if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then |
| gcloud compute addresses create "${MASTER_NAME}-internal-ip" --project "${PROJECT}" --region "$REGION" --subnet "$SUBNETWORK" |
| MASTER_INTERNAL_IP=$(gcloud compute addresses describe "${MASTER_NAME}-internal-ip" --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') |
| echo "Master internal ip is: $MASTER_INTERNAL_IP" |
| KUBERNETES_MASTER_NAME="${MASTER_INTERNAL_IP}" |
| MASTER_ADVERTISE_ADDRESS="${MASTER_INTERNAL_IP}" |
| fi |
| |
| create-certs "${MASTER_RESERVED_IP}" "${MASTER_INTERNAL_IP}" |
| create-etcd-certs "${MASTER_NAME}" |
| create-etcd-apiserver-certs "etcd-${MASTER_NAME}" "${MASTER_NAME}" |
| |
| if [[ "$(get-num-nodes)" -ge "50" ]]; then |
| # We block on master creation for large clusters to avoid doing too much |
| # unnecessary work in case master start-up fails (like creation of nodes). |
| create-master-instance "${MASTER_RESERVED_IP}" "${MASTER_INTERNAL_IP}" |
| else |
| create-master-instance "${MASTER_RESERVED_IP}" "${MASTER_INTERNAL_IP}" & |
| fi |
| |
| } |
| |
| # Adds master replica to etcd cluster. |
| # |
| # Assumed vars: |
| # REPLICA_NAME |
| # PROJECT |
| # EXISTING_MASTER_NAME |
| # EXISTING_MASTER_ZONE |
| # |
| # $1: etcd client port |
| # $2: etcd internal port |
| # $3: whether etcd communication should use mtls |
| # returns the result of ssh command which adds replica |
| function add-replica-to-etcd() { |
| local -r client_port="${1}" |
| local -r internal_port="${2}" |
| local -r use_mtls="${3}" |
| |
| TLSARG="" |
| PROTO="http://" |
| if [[ "${use_mtls}" == "true" ]]; then |
| # Keep in sync with ETCD_APISERVER_CA_CERT_PATH, ETCD_APISERVER_CLIENT_CERT_PATH and ETCD_APISERVER_CLIENT_KEY_PATH in configure-helper.sh. |
| TLSARG="--cacert /etc/srv/kubernetes/pki/etcd-apiserver-ca.crt --cert /etc/srv/kubernetes/pki/etcd-apiserver-client.crt --key /etc/srv/kubernetes/pki/etcd-apiserver-client.key" |
| PROTO="https://" |
| fi |
| run-gcloud-command "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" "curl ${TLSARG} ${PROTO}127.0.0.1:${client_port}/v2/members -XPOST -H \"Content-Type: application/json\" -d '{\"peerURLs\":[\"https://${REPLICA_NAME}:${internal_port}\"]}' -s" |
| return $? |
| } |
| |
| # Sets EXISTING_MASTER_NAME and EXISTING_MASTER_ZONE variables. |
| # |
| # Assumed vars: |
| # PROJECT |
| # |
| # NOTE: Must be in sync with get-replica-name-regexp |
| function set-existing-master() { |
| local existing_master |
| existing_master=$(gcloud compute instances list \ |
| --project "${PROJECT}" \ |
| --filter "name ~ '$(get-replica-name-regexp)'" \ |
| --format "value(name,zone)" | head -n1) |
| EXISTING_MASTER_NAME="$(echo "${existing_master}" | cut -f1)" |
| EXISTING_MASTER_ZONE="$(echo "${existing_master}" | cut -f2)" |
| } |
| |
| function replicate-master() { |
| set-replica-name |
| set-existing-master |
| |
| echo "Experimental: replicating existing master ${EXISTING_MASTER_ZONE}/${EXISTING_MASTER_NAME} as ${ZONE}/${REPLICA_NAME}" |
| |
| # Before we do anything else, we should configure etcd to expect more replicas. |
| if ! add-replica-to-etcd 2379 2380 true; then |
| echo "Failed to add master replica to etcd cluster." |
| return 1 |
| fi |
| if ! add-replica-to-etcd 4002 2381 false; then |
| echo "Failed to add master replica to etcd events cluster." |
| return 1 |
| fi |
| |
| # We have to make sure the disk is created before creating the master VM, so |
| # run this in the foreground. |
| gcloud compute disks create "${REPLICA_NAME}-pd" \ |
| --project "${PROJECT}" \ |
| --zone "${ZONE}" \ |
| --type "${MASTER_DISK_TYPE}" \ |
| --size "${MASTER_DISK_SIZE}" |
| |
| local existing_master_replicas |
| existing_master_replicas="$(get-all-replica-names)" |
| replicate-master-instance "${EXISTING_MASTER_ZONE}" "${EXISTING_MASTER_NAME}" "${existing_master_replicas}" |
| |
| # Add new replica to the load balancer. |
| gcloud compute target-pools add-instances "${MASTER_NAME}" \ |
| --project "${PROJECT}" \ |
| --zone "${ZONE}" \ |
| --instances "${REPLICA_NAME}" |
| |
| if [[ "${GCE_PRIVATE_CLUSTER:-}" == "true" ]]; then |
| add-to-internal-loadbalancer "${REPLICA_NAME}" "${ZONE}" |
| fi |
| } |
| |
| # Detaches old and ataches new external IP to a VM. |
| # |
| # Arguments: |
| # $1 - VM name |
| # $2 - VM zone |
| # $3 - external static IP; if empty will use an ephemeral IP address. |
| function attach-external-ip() { |
| local NAME=${1} |
| local ZONE=${2} |
| local IP_ADDR=${3:-} |
| local ACCESS_CONFIG_NAME |
| ACCESS_CONFIG_NAME=$(gcloud compute instances describe "${NAME}" \ |
| --project "${PROJECT}" --zone "${ZONE}" \ |
| --format="value(networkInterfaces[0].accessConfigs[0].name)") |
| gcloud compute instances delete-access-config "${NAME}" \ |
| --project "${PROJECT}" --zone "${ZONE}" \ |
| --access-config-name "${ACCESS_CONFIG_NAME}" |
| if [[ -z "${IP_ADDR}" ]]; then |
| gcloud compute instances add-access-config "${NAME}" \ |
| --project "${PROJECT}" --zone "${ZONE}" \ |
| --access-config-name "${ACCESS_CONFIG_NAME}" |
| else |
| gcloud compute instances add-access-config "${NAME}" \ |
| --project "${PROJECT}" --zone "${ZONE}" \ |
| --access-config-name "${ACCESS_CONFIG_NAME}" \ |
| --address "${IP_ADDR}" |
| fi |
| } |
| |
| # Creates load balancer in front of apiserver if it doesn't exists already. Assumes there's only one |
| # existing master replica. |
| # |
| # Assumes: |
| # PROJECT |
| # MASTER_NAME |
| # ZONE |
| # REGION |
| function create-loadbalancer() { |
| # Step 0: Return early if LB is already configured. |
| if gcloud compute forwarding-rules describe "${MASTER_NAME}" \ |
| --project "${PROJECT}" --region "${REGION}" > /dev/null 2>&1; then |
| echo "Load balancer already exists" |
| return |
| fi |
| |
| local EXISTING_MASTER_NAME |
| local EXISTING_MASTER_ZONE |
| EXISTING_MASTER_NAME="$(get-all-replica-names)" |
| EXISTING_MASTER_ZONE=$(gcloud compute instances list "${EXISTING_MASTER_NAME}" \ |
| --project "${PROJECT}" --format='value(zone)') |
| |
| echo "Creating load balancer in front of an already existing master in ${EXISTING_MASTER_ZONE}" |
| |
| # Step 1: Detach master IP address and attach ephemeral address to the existing master |
| attach-external-ip "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" |
| |
| # Step 2: Create target pool. |
| gcloud compute target-pools create "${MASTER_NAME}" --project "${PROJECT}" --region "${REGION}" |
| # TODO: We should also add master instances with suffixes |
| gcloud compute target-pools add-instances "${MASTER_NAME}" --instances "${EXISTING_MASTER_NAME}" --project "${PROJECT}" --zone "${EXISTING_MASTER_ZONE}" |
| |
| # Step 3: Create forwarding rule. |
| # TODO: This step can take up to 20 min. We need to speed this up... |
| gcloud compute forwarding-rules create "${MASTER_NAME}" \ |
| --project "${PROJECT}" --region "${REGION}" \ |
| --target-pool "${MASTER_NAME}" --address="${KUBE_MASTER_IP}" --ports=443 |
| |
| echo -n "Waiting for the load balancer configuration to propagate..." |
| local counter=0 |
| until curl -k -m1 "https://${KUBE_MASTER_IP}" &> /dev/null; do |
| counter=$((counter+1)) |
| echo -n . |
| if [[ ${counter} -ge 1800 ]]; then |
| echo -e "${color_red}TIMEOUT${color_norm}" >&2 |
| echo -e "${color_red}Load balancer failed to initialize within ${counter} seconds.${color_norm}" >&2 |
| exit 2 |
| fi |
| done |
| echo "DONE" |
| } |
| |
| |
| # attach-internal-master-ip attach internal ip to existing master. |
| # |
| # Assumes: |
| # * PROJECT |
| function attach-internal-master-ip() { |
| local name="${1}" |
| local zone="${2}" |
| local ip="${3}" |
| |
| local aliases |
| aliases=$(gcloud compute instances describe "${name}" --project "${PROJECT}" --zone "${zone}" --flatten='networkInterfaces[0].aliasIpRanges[]' --format='value[separator=':'](networkInterfaces[0].aliasIpRanges.subnetworkRangeName,networkInterfaces[0].aliasIpRanges.ipCidrRange)' | sed 's/^://' | paste -s -d';' -) |
| aliases="${aliases:+${aliases};}${ip}/32" |
| echo "Setting ${name}'s aliases to '${aliases}' (added ${ip})" |
| # Attach ${ip} to ${name} |
| gcloud compute instances network-interfaces update "${name}" --project "${PROJECT}" --zone "${zone}" --aliases="${aliases}" |
| gcloud compute instances add-metadata "${name}" --zone "${zone}" --metadata=kube-master-internal-ip="${ip}" |
| run-gcloud-command "${name}" "${zone}" 'sudo /bin/bash /home/kubernetes/bin/kube-master-internal-route.sh' || true |
| return $? |
| } |
| |
| |
| # detach-internal-master-ip detaches internal ip from existing master. |
| # |
| # Assumes: |
| # * PROJECT |
| function detach-internal-master-ip() { |
| local name="${1}" |
| local zone="${2}" |
| local ip="${3}" |
| |
| local aliases |
| aliases=$(gcloud compute instances describe "${name}" --project "${PROJECT}" --zone "${zone}" --flatten='networkInterfaces[0].aliasIpRanges[]' --format='value[separator=':'](networkInterfaces[0].aliasIpRanges.subnetworkRangeName,networkInterfaces[0].aliasIpRanges.ipCidrRange)' | sed 's/^://' | grep -v "${ip}" | paste -s -d';' -) |
| echo "Setting ${name}'s aliases to '${aliases}' (removed ${ip})" |
| # Detach ${MASTER_NAME}-internal-ip from ${name} |
| gcloud compute instances network-interfaces update "${name}" --project "${PROJECT}" --zone "${zone}" --aliases="${aliases}" |
| gcloud compute instances remove-metadata "${name}" --zone "${zone}" --keys=kube-master-internal-ip |
| # We want `ip route` to be run in the cloud and not this host |
| run-gcloud-command "${name}" "${zone}" "sudo ip route del to local ${ip}/32 dev \$(ip route | grep default | while read -r _ _ _ _ dev _; do echo \$dev; done)" || true |
| return $? |
| } |
| |
| # create-internal-loadbalancer creates an internal load balacer in front of existing master. |
| # |
| # Assumes: |
| # * MASTER_NAME |
| # * PROJECT |
| # * REGION |
| function create-internal-loadbalancer() { |
| if gcloud compute forwarding-rules describe "${MASTER_NAME}-internal" \ |
| --project "${PROJECT}" --region "${REGION}" > /dev/null 2>&1; then |
| echo "Load balancer already exists" |
| return |
| fi |
| |
| local EXISTING_MASTER_NAME |
| local EXISTING_MASTER_ZONE |
| EXISTING_MASTER_NAME="$(get-all-replica-names)" |
| EXISTING_MASTER_ZONE=$(gcloud compute instances list "${EXISTING_MASTER_NAME}" \ |
| --project "${PROJECT}" --format='value(zone)') |
| |
| echo "Detaching ${KUBE_MASTER_INTERNAL_IP} from ${EXISTING_MASTER_NAME}/${EXISTING_MASTER_ZONE}" |
| detach-internal-master-ip "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" "${KUBE_MASTER_INTERNAL_IP}" |
| |
| echo "Creating internal load balancer with IP: ${KUBE_MASTER_INTERNAL_IP}" |
| gcloud compute health-checks --project "${PROJECT}" create tcp "${MASTER_NAME}-hc" --port=443 |
| |
| gcloud compute backend-services create "${MASTER_NAME}" \ |
| --project "${PROJECT}" \ |
| --region "${REGION}" \ |
| --protocol tcp \ |
| --region "${REGION}" \ |
| --load-balancing-scheme internal \ |
| --health-checks "${MASTER_NAME}-hc" |
| |
| gcloud compute forwarding-rules create "${MASTER_NAME}-internal" \ |
| --project "${PROJECT}" \ |
| --region "${REGION}" \ |
| --load-balancing-scheme internal \ |
| --network "${NETWORK}" \ |
| --subnet "${SUBNETWORK}" \ |
| --address "${KUBE_MASTER_INTERNAL_IP}" \ |
| --ip-protocol TCP \ |
| --ports 443 \ |
| --backend-service "${MASTER_NAME}" \ |
| --backend-service-region "${REGION}" |
| |
| echo "Adding ${EXISTING_MASTER_NAME}/${EXISTING_MASTER_ZONE} to the load balancer" |
| add-to-internal-loadbalancer "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" |
| } |
| |
| # add-to-internal-loadbalancer adds an instance to ILB. |
| # Assumes: |
| # * MASTER_NAME |
| # * PROJECT |
| # * REGION |
| function add-to-internal-loadbalancer() { |
| local name="${1}" |
| local zone="${2}" |
| |
| gcloud compute instance-groups unmanaged create "${name}" --project "${PROJECT}" --zone "${zone}" |
| gcloud compute instance-groups unmanaged add-instances "${name}" --project "${PROJECT}" --zone "${zone}" --instances "${name}" |
| gcloud compute backend-services add-backend "${MASTER_NAME}" \ |
| --project "${PROJECT}" \ |
| --region "${REGION}" \ |
| --instance-group "${name}" \ |
| --instance-group-zone "${zone}" |
| } |
| |
| # remove-from-internal-loadbalancer removes an instance from ILB. |
| # Assumes: |
| # * MASTER_NAME |
| # * PROJECT |
| # * REGION |
| function remove-from-internal-loadbalancer() { |
| local name="${1}" |
| local zone="${2}" |
| |
| if gcloud compute instance-groups unmanaged describe "${name}" --project "${PROJECT}" --zone "${zone}" &>/dev/null; then |
| gcloud compute backend-services remove-backend "${MASTER_NAME}" \ |
| --project "${PROJECT}" \ |
| --region "${REGION}" \ |
| --instance-group "${name}" \ |
| --instance-group-zone "${zone}" |
| gcloud compute instance-groups unmanaged delete "${name}" --project "${PROJECT}" --zone "${zone}" --quiet |
| fi |
| } |
| |
| function delete-internal-loadbalancer() { |
| if gcloud compute forwarding-rules describe "${MASTER_NAME}-internal" --project "${PROJECT}" --region "${REGION}" &>/dev/null; then |
| gcloud compute forwarding-rules delete "${MASTER_NAME}-internal" --project "${PROJECT}" --region "${REGION}" --quiet |
| fi |
| |
| if gcloud compute backend-services describe "${MASTER_NAME}" --project "${PROJECT}" --region "${REGION}" &>/dev/null; then |
| gcloud compute backend-services delete "${MASTER_NAME}" --project "${PROJECT}" --region "${REGION}" --quiet |
| fi |
| if gcloud compute health-checks describe "${MASTER_NAME}-gc" --project "${PROJECT}" &>/dev/null; then |
| gcloud compute health-checks delete "${MASTER_NAME}-gc" --project "${PROJECT}" --quiet |
| fi |
| } |
| |
| function create-nodes-firewall() { |
| # Create a single firewall rule for all minions. |
| create-firewall-rule "${NODE_TAG}-all" "${CLUSTER_IP_RANGE}" "${NODE_TAG}" & |
| |
| # Report logging choice (if any). |
| if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]]; then |
| echo "+++ Logging using Fluentd to ${LOGGING_DESTINATION:-unknown}" |
| fi |
| |
| # Wait for last batch of jobs |
| kube::util::wait-for-jobs || { |
| code=$? |
| echo -e "${color_red}Failed to create firewall rule.${color_norm}" >&2 |
| exit $code |
| } |
| } |
| |
| function get-scope-flags() { |
| local scope_flags= |
| if [[ -n "${NODE_SCOPES}" ]]; then |
| scope_flags="--scopes ${NODE_SCOPES}" |
| else |
| scope_flags="--no-scopes" |
| fi |
| echo "${scope_flags}" |
| } |
| |
| function create-nodes-template() { |
| echo "Creating nodes." |
| |
| local scope_flags |
| scope_flags=$(get-scope-flags) |
| |
| write-linux-node-env |
| write-windows-node-env |
| |
| # NOTE: these template names and their format must match |
| # create-[linux,windows]-nodes() as well as get-template()! |
| local linux_template_name="${NODE_INSTANCE_PREFIX}-template" |
| local windows_template_name="${WINDOWS_NODE_INSTANCE_PREFIX}-template" |
| create-linux-node-instance-template "$linux_template_name" |
| create-windows-node-instance-template "$windows_template_name" "${scope_flags[*]}" |
| if [[ -n "${ADDITIONAL_MACHINE_TYPE:-}" ]]; then |
| local linux_extra_template_name="${NODE_INSTANCE_PREFIX}-extra-template" |
| create-linux-node-instance-template "$linux_extra_template_name" "${ADDITIONAL_MACHINE_TYPE}" |
| fi |
| } |
| |
| # Assumes: |
| # - MAX_INSTANCES_PER_MIG |
| # - NUM_NODES |
| # - NUM_WINDOWS_NODES |
| # exports: |
| # - NUM_MIGS |
| # - NUM_WINDOWS_MIGS |
| function set_num_migs() { |
| local defaulted_max_instances_per_mig=${MAX_INSTANCES_PER_MIG:-1000} |
| |
| if [[ ${defaulted_max_instances_per_mig} -le "0" ]]; then |
| echo "MAX_INSTANCES_PER_MIG cannot be negative. Assuming default 1000" |
| defaulted_max_instances_per_mig=1000 |
| fi |
| export NUM_MIGS=$(((NUM_NODES + defaulted_max_instances_per_mig - 1) / defaulted_max_instances_per_mig)) |
| export NUM_WINDOWS_MIGS=$(((NUM_WINDOWS_NODES + defaulted_max_instances_per_mig - 1) / defaulted_max_instances_per_mig)) |
| } |
| |
| # Assumes: |
| # - NUM_MIGS |
| # - NODE_INSTANCE_PREFIX |
| # - NUM_NODES |
| # - PROJECT |
| # - ZONE |
| function create-linux-nodes() { |
| local template_name="${NODE_INSTANCE_PREFIX}-template" |
| local extra_template_name="${NODE_INSTANCE_PREFIX}-extra-template" |
| |
| local nodes="${NUM_NODES}" |
| if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then |
| echo "Creating a special node for heapster with machine-type ${HEAPSTER_MACHINE_TYPE}" |
| create-heapster-node |
| nodes=$(( nodes - 1 )) |
| fi |
| |
| if [[ -n "${ADDITIONAL_MACHINE_TYPE:-}" && "${NUM_ADDITIONAL_NODES:-}" -gt 0 ]]; then |
| local num_additional="${NUM_ADDITIONAL_NODES}" |
| if [[ "${NUM_ADDITIONAL_NODES:-}" -gt "${nodes}" ]]; then |
| echo "Capping NUM_ADDITIONAL_NODES to ${nodes}" |
| num_additional="${nodes}" |
| fi |
| if [[ "${num_additional:-}" -gt 0 ]]; then |
| echo "Creating ${num_additional} special nodes with machine-type ${ADDITIONAL_MACHINE_TYPE}" |
| local extra_group_name="${NODE_INSTANCE_PREFIX}-extra" |
| gcloud compute instance-groups managed \ |
| create "${extra_group_name}" \ |
| --project "${PROJECT}" \ |
| --zone "${ZONE}" \ |
| --base-instance-name "${extra_group_name}" \ |
| --size "${num_additional}" \ |
| --template "${extra_template_name}" || true; |
| gcloud compute instance-groups managed wait-until --stable \ |
| "${extra_group_name}" \ |
| --zone "${ZONE}" \ |
| --project "${PROJECT}" \ |
| --timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true |
| nodes=$(( nodes - num_additional )) |
| fi |
| fi |
| |
| local instances_left=${nodes} |
| |
| for ((i=1; i<=NUM_MIGS; i++)); do |
| local group_name="${NODE_INSTANCE_PREFIX}-group-$i" |
| if [[ $i -eq ${NUM_MIGS} ]]; then |
| # TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG. |
| # We should change it at some point, but note #18545 when changing this. |
| group_name="${NODE_INSTANCE_PREFIX}-group" |
| fi |
| # Spread the remaining number of nodes evenly |
| this_mig_size=$((instances_left / (NUM_MIGS - i + 1))) |
| instances_left=$((instances_left - this_mig_size)) |
| |
| # Run instance-groups creation in parallel. |
| { |
| gcloud compute instance-groups managed \ |
| create "${group_name}" \ |
| --project "${PROJECT}" \ |
| --zone "${ZONE}" \ |
| --base-instance-name "${group_name}" \ |
| --size "${this_mig_size}" \ |
| --template "${template_name}" || true; |
| gcloud compute instance-groups managed wait-until --stable \ |
| "${group_name}" \ |
| --zone "${ZONE}" \ |
| --project "${PROJECT}" \ |
| --timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true |
| } & |
| done |
| wait |
| } |
| |
| # Assumes: |
| # - NUM_WINDOWS_MIGS |
| # - WINDOWS_NODE_INSTANCE_PREFIX |
| # - NUM_WINDOWS_NODES |
| # - PROJECT |
| # - ZONE |
| function create-windows-nodes() { |
| local template_name="${WINDOWS_NODE_INSTANCE_PREFIX}-template" |
| |
| local -r nodes="${NUM_WINDOWS_NODES}" |
| local instances_left=${nodes} |
| |
| for ((i=1; i <= NUM_WINDOWS_MIGS; i++)); do |
| local group_name="${WINDOWS_NODE_INSTANCE_PREFIX}-group-$i" |
| if [[ $i -eq ${NUM_WINDOWS_MIGS} ]]; then |
| # TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG. |
| # We should change it at some point, but note #18545 when changing this. |
| group_name="${WINDOWS_NODE_INSTANCE_PREFIX}-group" |
| fi |
| # Spread the remaining number of nodes evenly |
| this_mig_size=$((instances_left / (NUM_WINDOWS_MIGS - i + 1))) |
| instances_left=$((instances_left - this_mig_size)) |
| |
| gcloud compute instance-groups managed \ |
| create "${group_name}" \ |
| --project "${PROJECT}" \ |
| --zone "${ZONE}" \ |
| --base-instance-name "${group_name}" \ |
| --size "${this_mig_size}" \ |
| --template "${template_name}" || true; |
| gcloud compute instance-groups managed wait-until --stable \ |
| "${group_name}" \ |
| --zone "${ZONE}" \ |
| --project "${PROJECT}" \ |
| --timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true; |
| done |
| } |
| |
| # Assumes: |
| # - NODE_INSTANCE_PREFIX |
| # - PROJECT |
| # - NETWORK_PROJECT |
| # - REGION |
| # - ZONE |
| # - HEAPSTER_MACHINE_TYPE |
| # - NODE_DISK_TYPE |
| # - NODE_DISK_SIZE |
| # - NODE_IMAGE_PROJECT |
| # - NODE_IMAGE |
| # - NODE_SERVICE_ACCOUNT |
| # - NODE_TAG |
| # - NETWORK |
| # - ENABLE_IP_ALIASES |
| # - SUBNETWORK |
| # - IP_ALIAS_SIZE |
| function create-heapster-node() { |
| local gcloud="gcloud" |
| |
| local network |
| network=$(make-gcloud-network-argument \ |
| "${NETWORK_PROJECT}" \ |
| "${REGION}" \ |
| "${NETWORK}" \ |
| "${SUBNETWORK:-}" \ |
| "" \ |
| "${ENABLE_IP_ALIASES:-}" \ |
| "${IP_ALIAS_SIZE:-}") |
| |
| # Deliberately word split ${network} and $(get-scope-flags) |
| # shellcheck disable=SC2086 disable=SC2046 |
| ${gcloud} compute instances \ |
| create "${NODE_INSTANCE_PREFIX}-heapster" \ |
| --project "${PROJECT}" \ |
| --zone "${ZONE}" \ |
| --machine-type="${HEAPSTER_MACHINE_TYPE}" \ |
| --boot-disk-type "${NODE_DISK_TYPE}" \ |
| --boot-disk-size "${NODE_DISK_SIZE}" \ |
| --image-project="${NODE_IMAGE_PROJECT}" \ |
| --image "${NODE_IMAGE}" \ |
| --service-account "${NODE_SERVICE_ACCOUNT}" \ |
| --tags "${NODE_TAG}" \ |
| ${network} \ |
| $(get-scope-flags) \ |
| --metadata-from-file "$(get-node-instance-metadata-from-file "heapster-kube-env")" |
| } |
| |
| # Assumes: |
| # - NUM_MIGS |
| # - NODE_INSTANCE_PREFIX |
| # - PROJECT |
| # - ZONE |
| # - AUTOSCALER_MAX_NODES |
| # - AUTOSCALER_MIN_NODES |
| # Exports |
| # - AUTOSCALER_MIG_CONFIG |
| function create-cluster-autoscaler-mig-config() { |
| |
| # Each MIG must have at least one node, so the min number of nodes |
| # must be greater or equal to the number of migs. |
| if [[ ${AUTOSCALER_MIN_NODES} -lt 0 ]]; then |
| echo "AUTOSCALER_MIN_NODES must be greater or equal 0" |
| exit 2 |
| fi |
| |
| # Each MIG must have at least one node, so the min number of nodes |
| # must be greater or equal to the number of migs. |
| if [[ ${AUTOSCALER_MAX_NODES} -lt ${NUM_MIGS} ]]; then |
| echo "AUTOSCALER_MAX_NODES must be greater or equal ${NUM_MIGS}" |
| exit 2 |
| fi |
| if [[ ${NUM_WINDOWS_MIGS} -gt 0 ]]; then |
| # TODO(pjh): implement Windows support in this function. |
| echo "Not implemented yet: autoscaler config for Windows MIGs" |
| exit 2 |
| fi |
| |
| # The code assumes that the migs were created with create-nodes |
| # function which tries to evenly spread nodes across the migs. |
| AUTOSCALER_MIG_CONFIG="" |
| |
| local left_min=${AUTOSCALER_MIN_NODES} |
| local left_max=${AUTOSCALER_MAX_NODES} |
| |
| for ((i=1; i <= NUM_MIGS; i++)); do |
| local group_name="${NODE_INSTANCE_PREFIX}-group-$i" |
| if [[ $i -eq ${NUM_MIGS} ]]; then |
| # TODO: We don't add a suffix for the last group to keep backward compatibility when there's only one MIG. |
| # We should change it at some point, but note #18545 when changing this. |
| group_name="${NODE_INSTANCE_PREFIX}-group" |
| fi |
| |
| this_mig_min=$((left_min/(NUM_MIGS-i+1))) |
| this_mig_max=$((left_max/(NUM_MIGS-i+1))) |
| left_min=$((left_min-this_mig_min)) |
| left_max=$((left_max-this_mig_max)) |
| |
| local mig_url="https://www.googleapis.com/compute/v1/projects/${PROJECT}/zones/${ZONE}/instanceGroups/${group_name}" |
| AUTOSCALER_MIG_CONFIG="${AUTOSCALER_MIG_CONFIG} --nodes=${this_mig_min}:${this_mig_max}:${mig_url}" |
| done |
| |
| AUTOSCALER_MIG_CONFIG="${AUTOSCALER_MIG_CONFIG} --scale-down-enabled=${AUTOSCALER_ENABLE_SCALE_DOWN}" |
| } |
| |
| # Assumes: |
| # - NUM_MIGS |
| # - NODE_INSTANCE_PREFIX |
| # - PROJECT |
| # - ZONE |
| # - ENABLE_CLUSTER_AUTOSCALER |
| # - AUTOSCALER_MAX_NODES |
| # - AUTOSCALER_MIN_NODES |
| function create-autoscaler-config() { |
| # Create autoscaler for nodes configuration if requested |
| if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then |
| create-cluster-autoscaler-mig-config |
| echo "Using autoscaler config: ${AUTOSCALER_MIG_CONFIG} ${AUTOSCALER_EXPANDER_CONFIG}" |
| fi |
| } |
| |
| function check-cluster() { |
| detect-node-names |
| detect-master |
| |
| echo "Waiting up to ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds for cluster initialization." |
| echo |
| echo " This will continually check to see if the API for kubernetes is reachable." |
| echo " This may time out if there was some uncaught error during start up." |
| echo |
| |
| # curl in mavericks is borked. |
| secure="" |
| if which sw_vers >& /dev/null; then |
| if [[ $(sw_vers | grep ProductVersion | awk '{print $2}') = "10.9."* ]]; then |
| secure="--insecure" |
| fi |
| fi |
| |
| local start_time |
| local curl_out |
| start_time=$(date +%s) |
| curl_out=$(mktemp) |
| kube::util::trap_add "rm -f ${curl_out}" EXIT |
| until curl -vsS --cacert "${CERT_DIR}/pki/ca.crt" \ |
| -H "Authorization: Bearer ${KUBE_BEARER_TOKEN}" \ |
| ${secure} \ |
| --max-time 5 --fail \ |
| "https://${KUBE_MASTER_IP}/api/v1/pods?limit=100" > "${curl_out}" 2>&1; do |
| local elapsed |
| elapsed=$(($(date +%s) - start_time)) |
| if [[ ${elapsed} -gt ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} ]]; then |
| echo -e "${color_red}Cluster failed to initialize within ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds.${color_norm}" >&2 |
| echo "Last output from querying API server follows:" >&2 |
| echo "-----------------------------------------------------" >&2 |
| cat "${curl_out}" >&2 |
| echo "-----------------------------------------------------" >&2 |
| exit 2 |
| fi |
| printf "." |
| sleep 2 |
| done |
| |
| echo "Kubernetes cluster created." |
| |
| export KUBE_CERT="${CERT_DIR}/pki/issued/kubecfg.crt" |
| export KUBE_KEY="${CERT_DIR}/pki/private/kubecfg.key" |
| export CA_CERT="${CERT_DIR}/pki/ca.crt" |
| export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}" |
| ( |
| umask 077 |
| |
| # Update the user's kubeconfig to include credentials for this apiserver. |
| create-kubeconfig |
| ) |
| |
| # ensures KUBECONFIG is set |
| get-kubeconfig-basicauth |
| |
| if [[ ${GCE_UPLOAD_KUBCONFIG_TO_MASTER_METADATA:-} == "true" ]]; then |
| gcloud compute instances add-metadata "${MASTER_NAME}" --project="${PROJECT}" --zone="${ZONE}" --metadata-from-file="kubeconfig=${KUBECONFIG}" || true |
| fi |
| |
| echo |
| echo -e "${color_green:-}Kubernetes cluster is running. The master is running at:" |
| echo |
| echo -e "${color_yellow} https://${KUBE_MASTER_IP}" |
| echo |
| echo -e "${color_green}The user name and password to use is located in ${KUBECONFIG}.${color_norm}" |
| echo |
| |
| } |
| |
| # Removes master replica from etcd cluster. |
| # |
| # Assumed vars: |
| # REPLICA_NAME |
| # PROJECT |
| # EXISTING_MASTER_NAME |
| # EXISTING_MASTER_ZONE |
| # |
| # $1: etcd client port |
| # $2: whether etcd communication should use mtls |
| # returns the result of ssh command which removes replica |
| function remove-replica-from-etcd() { |
| local -r port="${1}" |
| local -r use_mtls="${2}" |
| |
| TLSARG="" |
| PROTO="http://" |
| if [[ "${use_mtls}" == "true" ]]; then |
| # Keep in sync with ETCD_APISERVER_CA_CERT_PATH, ETCD_APISERVER_CLIENT_CERT_PATH and ETCD_APISERVER_CLIENT_KEY_PATH in configure-helper.sh. |
| TLSARG="--cacert /etc/srv/kubernetes/pki/etcd-apiserver-ca.crt --cert /etc/srv/kubernetes/pki/etcd-apiserver-client.crt --key /etc/srv/kubernetes/pki/etcd-apiserver-client.key" |
| PROTO="https://" |
| fi |
| [[ -n "${EXISTING_MASTER_NAME}" ]] || return |
| run-gcloud-command "${EXISTING_MASTER_NAME}" "${EXISTING_MASTER_ZONE}" "curl -s ${TLSARG} ${PROTO}127.0.0.1:${port}/v2/members/\$(curl -s ${TLSARG} ${PROTO}127.0.0.1:${port}/v2/members -XGET | sed 's/{\\\"id/\n/g' | grep ${REPLICA_NAME}\\\" | cut -f 3 -d \\\") -XDELETE -L 2>/dev/null" |
| local -r res=$? |
| echo "Removing etcd replica, name: ${REPLICA_NAME}, port: ${port}, result: ${res}" |
| return "${res}" |
| } |
| |
| # Delete a kubernetes cluster. This is called from test-teardown. |
| # |
| # Assumed vars: |
| # MASTER_NAME |
| # NODE_INSTANCE_PREFIX |
| # WINDOWS_NODE_INSTANCE_PREFIX |
| # ZONE |
| # This function tears down cluster resources 10 at a time to avoid issuing too many |
| # API calls and exceeding API quota. It is important to bring down the instances before bringing |
| # down the firewall rules and routes. |
| function kube-down() { |
| local -r batch=200 |
| |
| detect-project |
| detect-node-names # For INSTANCE_GROUPS and WINDOWS_INSTANCE_GROUPS |
| |
| echo "Bringing down cluster" |
| set +e # Do not stop on error |
| |
| if [[ "${KUBE_DELETE_NODES:-}" != "false" ]]; then |
| # Get the name of the managed instance group template before we delete the |
| # managed instance group. (The name of the managed instance group template may |
| # change during a cluster upgrade.) |
| local templates |
| templates=$(get-template "${PROJECT}") |
| |
| # Deliberately allow globbing, do not change unless a bug is found |
| # shellcheck disable=SC2206 |
| local all_instance_groups=(${INSTANCE_GROUPS[@]:-} ${WINDOWS_INSTANCE_GROUPS[@]:-}) |
| # Deliberately do not quote, do not change unless a bug is found |
| # shellcheck disable=SC2068 |
| for group in ${all_instance_groups[@]:-}; do |
| { |
| if gcloud compute instance-groups managed describe "${group}" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then |
| gcloud compute instance-groups managed delete \ |
| --project "${PROJECT}" \ |
| --quiet \ |
| --zone "${ZONE}" \ |
| "${group}" |
| fi |
| } & |
| done |
| |
| # Wait for last batch of jobs |
| kube::util::wait-for-jobs || { |
| echo -e "Failed to delete instance group(s)." >&2 |
| } |
| |
| # Deliberately do not quote, do not change unless a bug is found |
| # shellcheck disable=SC2068 |
| for template in ${templates[@]:-}; do |
| { |
| if gcloud compute instance-templates describe --project "${PROJECT}" "${template}" &>/dev/null; then |
| gcloud compute instance-templates delete \ |
| --project "${PROJECT}" \ |
| --quiet \ |
| "${template}" |
| fi |
| } & |
| done |
| |
| # Wait for last batch of jobs |
| kube::util::wait-for-jobs || { |
| echo -e "Failed to delete instance template(s)." >&2 |
| } |
| |
| # Delete the special heapster node (if it exists). |
| if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then |
| local -r heapster_machine_name="${NODE_INSTANCE_PREFIX}-heapster" |
| if gcloud compute instances describe "${heapster_machine_name}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then |
| # Now we can safely delete the VM. |
| gcloud compute instances delete \ |
| --project "${PROJECT}" \ |
| --quiet \ |
| --delete-disks all \ |
| --zone "${ZONE}" \ |
| "${heapster_machine_name}" |
| fi |
| fi |
| fi |
| |
| local -r REPLICA_NAME="${KUBE_REPLICA_NAME:-$(get-replica-name)}" |
| |
| set-existing-master |
| |
| # Un-register the master replica from etcd and events etcd. |
| remove-replica-from-etcd 2379 true |
| remove-replica-from-etcd 4002 false |
| |
| # Delete the master replica (if it exists). |
| if gcloud compute instances describe "${REPLICA_NAME}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then |
| # If there is a load balancer in front of apiservers we need to first update its configuration. |
| if gcloud compute target-pools describe "${MASTER_NAME}" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then |
| gcloud compute target-pools remove-instances "${MASTER_NAME}" \ |
| --project "${PROJECT}" \ |
| --zone "${ZONE}" \ |
| --instances "${REPLICA_NAME}" |
| fi |
| # Detach replica from LB if needed. |
| if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then |
| remove-from-internal-loadbalancer "${REPLICA_NAME}" "${ZONE}" |
| fi |
| # Now we can safely delete the VM. |
| gcloud compute instances delete \ |
| --project "${PROJECT}" \ |
| --quiet \ |
| --delete-disks all \ |
| --zone "${ZONE}" \ |
| "${REPLICA_NAME}" |
| fi |
| |
| # Delete the master replica pd (possibly leaked by kube-up if master create failed). |
| # TODO(jszczepkowski): remove also possibly leaked replicas' pds |
| local -r replica_pd="${REPLICA_NAME:-${MASTER_NAME}}-pd" |
| if gcloud compute disks describe "${replica_pd}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then |
| gcloud compute disks delete \ |
| --project "${PROJECT}" \ |
| --quiet \ |
| --zone "${ZONE}" \ |
| "${replica_pd}" |
| fi |
| |
| # Check if this are any remaining master replicas. |
| local REMAINING_MASTER_COUNT |
| REMAINING_MASTER_COUNT=$(gcloud compute instances list \ |
| --project "${PROJECT}" \ |
| --filter="name ~ '$(get-replica-name-regexp)'" \ |
| --format "value(zone)" | wc -l) |
| |
| # In the replicated scenario, if there's only a single master left, we should also delete load balancer in front of it. |
| if [[ "${REMAINING_MASTER_COUNT}" -eq 1 ]]; then |
| detect-master |
| local REMAINING_REPLICA_NAME |
| local REMAINING_REPLICA_ZONE |
| REMAINING_REPLICA_NAME="$(get-all-replica-names)" |
| REMAINING_REPLICA_ZONE=$(gcloud compute instances list "${REMAINING_REPLICA_NAME}" \ |
| --project "${PROJECT}" --format='value(zone)') |
| if gcloud compute forwarding-rules describe "${MASTER_NAME}" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then |
| gcloud compute forwarding-rules delete \ |
| --project "${PROJECT}" \ |
| --region "${REGION}" \ |
| --quiet \ |
| "${MASTER_NAME}" |
| attach-external-ip "${REMAINING_REPLICA_NAME}" "${REMAINING_REPLICA_ZONE}" "${KUBE_MASTER_IP}" |
| gcloud compute target-pools delete \ |
| --project "${PROJECT}" \ |
| --region "${REGION}" \ |
| --quiet \ |
| "${MASTER_NAME}" |
| fi |
| |
| if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then |
| remove-from-internal-loadbalancer "${REMAINING_REPLICA_NAME}" "${REMAINING_REPLICA_ZONE}" |
| delete-internal-loadbalancer |
| attach-internal-master-ip "${REMAINING_REPLICA_NAME}" "${REMAINING_REPLICA_ZONE}" "${KUBE_MASTER_INTERNAL_IP}" |
| fi |
| fi |
| |
| # If there are no more remaining master replicas, we should delete all remaining network resources. |
| if [[ "${REMAINING_MASTER_COUNT}" -eq 0 ]]; then |
| # Delete firewall rule for the master, etcd servers, and nodes. |
| delete-firewall-rules "${MASTER_NAME}-https" "${MASTER_NAME}-etcd" "${NODE_TAG}-all" "${MASTER_NAME}-konnectivity-server" |
| # Delete the master's reserved IP |
| if gcloud compute addresses describe "${MASTER_NAME}-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then |
| gcloud compute addresses delete \ |
| --project "${PROJECT}" \ |
| --region "${REGION}" \ |
| --quiet \ |
| "${MASTER_NAME}-ip" |
| fi |
| |
| if gcloud compute addresses describe "${MASTER_NAME}-internal-ip" --region "${REGION}" --project "${PROJECT}" &>/dev/null; then |
| gcloud compute addresses delete \ |
| --project "${PROJECT}" \ |
| --region "${REGION}" \ |
| --quiet \ |
| "${MASTER_NAME}-internal-ip" |
| fi |
| fi |
| |
| if [[ "${KUBE_DELETE_NODES:-}" != "false" ]]; then |
| # Find out what minions are running. |
| local -a minions |
| kube::util::read-array minions < <(gcloud compute instances list \ |
| --project "${PROJECT}" \ |
| --filter="(name ~ '${NODE_INSTANCE_PREFIX}-.+' OR name ~ '${WINDOWS_NODE_INSTANCE_PREFIX}-.+') AND zone:(${ZONE})" \ |
| --format='value(name)') |
| # If any minions are running, delete them in batches. |
| while (( "${#minions[@]}" > 0 )); do |
| echo Deleting nodes "${minions[*]::${batch}}" |
| gcloud compute instances delete \ |
| --project "${PROJECT}" \ |
| --quiet \ |
| --delete-disks boot \ |
| --zone "${ZONE}" \ |
| "${minions[@]::${batch}}" |
| minions=( "${minions[@]:${batch}}" ) |
| done |
| fi |
| |
| # If there are no more remaining master replicas: delete routes, pd for influxdb and update kubeconfig |
| if [[ "${REMAINING_MASTER_COUNT}" -eq 0 ]]; then |
| # Delete routes. |
| local -a routes |
| # Clean up all routes w/ names like "<cluster-name>-<node-GUID>" |
| # e.g. "kubernetes-12345678-90ab-cdef-1234-567890abcdef". The name is |
| # determined by the node controller on the master. |
| # Note that this is currently a noop, as synchronously deleting the node MIG |
| # first allows the master to cleanup routes itself. |
| local TRUNCATED_PREFIX="${INSTANCE_PREFIX:0:26}" |
| kube::util::read-array routes < <(gcloud compute routes list --project "${NETWORK_PROJECT}" \ |
| --filter="name ~ '${TRUNCATED_PREFIX}-.{8}-.{4}-.{4}-.{4}-.{12}'" \ |
| --format='value(name)') |
| while (( "${#routes[@]}" > 0 )); do |
| echo Deleting routes "${routes[*]::${batch}}" |
| gcloud compute routes delete \ |
| --project "${NETWORK_PROJECT}" \ |
| --quiet \ |
| "${routes[@]::${batch}}" |
| routes=( "${routes[@]:${batch}}" ) |
| done |
| |
| # Delete persistent disk for influx-db. |
| if gcloud compute disks describe "${INSTANCE_PREFIX}"-influxdb-pd --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then |
| gcloud compute disks delete \ |
| --project "${PROJECT}" \ |
| --quiet \ |
| --zone "${ZONE}" \ |
| "${INSTANCE_PREFIX}"-influxdb-pd |
| fi |
| |
| # Delete all remaining firewall rules and network. |
| delete-firewall-rules \ |
| "${CLUSTER_NAME}-default-internal-master" \ |
| "${CLUSTER_NAME}-default-internal-node" |
| |
| if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then |
| delete-firewall-rules \ |
| "${NETWORK}-default-ssh" \ |
| "${NETWORK}-default-rdp" \ |
| "${NETWORK}-default-internal" # Pre-1.5 clusters |
| delete-cloud-nat-router |
| # Delete all remaining firewall rules in the network. |
| delete-all-firewall-rules || true |
| delete-subnetworks || true |
| delete-network || true # might fail if there are leaked resources that reference the network |
| fi |
| |
| # If there are no more remaining master replicas, we should update kubeconfig. |
| export CONTEXT="${PROJECT}_${INSTANCE_PREFIX}" |
| clear-kubeconfig |
| else |
| # If some master replicas remain: cluster has been changed, we need to re-validate it. |
| echo "... calling validate-cluster" >&2 |
| # Override errexit |
| (validate-cluster) && validate_result="$?" || validate_result="$?" |
| |
| # We have two different failure modes from validate cluster: |
| # - 1: fatal error - cluster won't be working correctly |
| # - 2: weak error - something went wrong, but cluster probably will be working correctly |
| # We just print an error message in case 2). |
| if [[ "${validate_result}" -eq 1 ]]; then |
| exit 1 |
| elif [[ "${validate_result}" -eq 2 ]]; then |
| echo "...ignoring non-fatal errors in validate-cluster" >&2 |
| fi |
| fi |
| set -e |
| } |
| |
| # Prints name of one of the master replicas in the current zone. It will be either |
| # just MASTER_NAME or MASTER_NAME with a suffix for a replica (see get-replica-name-regexp). |
| # |
| # Assumed vars: |
| # PROJECT |
| # ZONE |
| # MASTER_NAME |
| # |
| # NOTE: Must be in sync with get-replica-name-regexp and set-replica-name. |
| function get-replica-name() { |
| # Ignore if gcloud compute fails and successfully echo any outcome |
| # shellcheck disable=SC2005 |
| echo "$(gcloud compute instances list \ |
| --project "${PROJECT}" \ |
| --filter="name ~ '$(get-replica-name-regexp)' AND zone:(${ZONE})" \ |
| --format "value(name)" | head -n1)" |
| } |
| |
| # Prints comma-separated names of all of the master replicas in all zones. |
| # |
| # Assumed vars: |
| # PROJECT |
| # MASTER_NAME |
| # |
| # NOTE: Must be in sync with get-replica-name-regexp and set-replica-name. |
| function get-all-replica-names() { |
| # Ignore if gcloud compute fails and successfully echo any outcome |
| # shellcheck disable=SC2005 |
| echo "$(gcloud compute instances list \ |
| --project "${PROJECT}" \ |
| --filter="name ~ '$(get-replica-name-regexp)'" \ |
| --format "value(name)" | tr "\n" "," | sed 's/,$//')" |
| } |
| |
| # Prints the number of all of the master replicas in all zones. |
| # |
| # Assumed vars: |
| # MASTER_NAME |
| function get-master-replicas-count() { |
| detect-project |
| local num_masters |
| num_masters=$(gcloud compute instances list \ |
| --project "${PROJECT}" \ |
| --filter="name ~ '$(get-replica-name-regexp)'" \ |
| --format "value(zone)" | wc -l) |
| echo -n "${num_masters}" |
| } |
| |
| # Prints regexp for full master machine name. In a cluster with replicated master, |
| # VM names may either be MASTER_NAME or MASTER_NAME with a suffix for a replica. |
| function get-replica-name-regexp() { |
| echo "^${MASTER_NAME}(-...)?" |
| } |
| |
| # Sets REPLICA_NAME to a unique name for a master replica that will match |
| # expected regexp (see get-replica-name-regexp). |
| # |
| # Assumed vars: |
| # PROJECT |
| # ZONE |
| # MASTER_NAME |
| # |
| # Sets: |
| # REPLICA_NAME |
| function set-replica-name() { |
| local instances |
| instances=$(gcloud compute instances list \ |
| --project "${PROJECT}" \ |
| --filter="name ~ '$(get-replica-name-regexp)'" \ |
| --format "value(name)") |
| |
| suffix="" |
| while echo "${instances}" | grep "${suffix}" &>/dev/null; do |
| suffix="$(date | md5sum | head -c3)" |
| done |
| REPLICA_NAME="${MASTER_NAME}-${suffix}" |
| } |
| |
| # Gets the instance templates in use by the cluster. It echos the template names |
| # so that the function output can be used. |
| # Assumed vars: |
| # NODE_INSTANCE_PREFIX |
| # WINDOWS_NODE_INSTANCE_PREFIX |
| # |
| # $1: project |
| function get-template() { |
| local linux_filter="${NODE_INSTANCE_PREFIX}-(extra-)?template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?" |
| local windows_filter="${WINDOWS_NODE_INSTANCE_PREFIX}-template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?" |
| |
| gcloud compute instance-templates list \ |
| --filter="name ~ '${linux_filter}' OR name ~ '${windows_filter}'" \ |
| --project="${1}" --format='value(name)' |
| } |
| |
| # Checks if there are any present resources related kubernetes cluster. |
| # |
| # Assumed vars: |
| # MASTER_NAME |
| # NODE_INSTANCE_PREFIX |
| # WINDOWS_NODE_INSTANCE_PREFIX |
| # ZONE |
| # REGION |
| # Vars set: |
| # KUBE_RESOURCE_FOUND |
| function check-resources() { |
| detect-project |
| detect-node-names |
| |
| echo "Looking for already existing resources" |
| KUBE_RESOURCE_FOUND="" |
| |
| if [[ -n "${INSTANCE_GROUPS[*]:-}" ]]; then |
| KUBE_RESOURCE_FOUND="Managed instance groups ${INSTANCE_GROUPS[*]}" |
| return 1 |
| fi |
| if [[ -n "${WINDOWS_INSTANCE_GROUPS[*]:-}" ]]; then |
| KUBE_RESOURCE_FOUND="Managed instance groups ${WINDOWS_INSTANCE_GROUPS[*]}" |
| return 1 |
| fi |
| |
| if gcloud compute instance-templates describe --project "${PROJECT}" "${NODE_INSTANCE_PREFIX}-template" &>/dev/null; then |
| KUBE_RESOURCE_FOUND="Instance template ${NODE_INSTANCE_PREFIX}-template" |
| return 1 |
| fi |
| if gcloud compute instance-templates describe --project "${PROJECT}" "${WINDOWS_NODE_INSTANCE_PREFIX}-template" &>/dev/null; then |
| KUBE_RESOURCE_FOUND="Instance template ${WINDOWS_NODE_INSTANCE_PREFIX}-template" |
| return 1 |
| fi |
| |
| if gcloud compute instances describe --project "${PROJECT}" "${MASTER_NAME}" --zone "${ZONE}" &>/dev/null; then |
| KUBE_RESOURCE_FOUND="Kubernetes master ${MASTER_NAME}" |
| return 1 |
| fi |
| |
| if gcloud compute disks describe --project "${PROJECT}" "${MASTER_NAME}"-pd --zone "${ZONE}" &>/dev/null; then |
| KUBE_RESOURCE_FOUND="Persistent disk ${MASTER_NAME}-pd" |
| return 1 |
| fi |
| |
| # Find out what minions are running. |
| local -a minions |
| kube::util::read-array minions < <(gcloud compute instances list \ |
| --project "${PROJECT}" \ |
| --filter="(name ~ '${NODE_INSTANCE_PREFIX}-.+' OR name ~ '${WINDOWS_NODE_INSTANCE_PREFIX}-.+') AND zone:(${ZONE})" \ |
| --format='value(name)') |
| if (( "${#minions[@]}" > 0 )); then |
| KUBE_RESOURCE_FOUND="${#minions[@]} matching ${NODE_INSTANCE_PREFIX}-.+ or ${WINDOWS_NODE_INSTANCE_PREFIX}-.+" |
| return 1 |
| fi |
| |
| if gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${MASTER_NAME}-https" &>/dev/null; then |
| KUBE_RESOURCE_FOUND="Firewall rules for ${MASTER_NAME}-https" |
| return 1 |
| fi |
| |
| if gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NODE_TAG}-all" &>/dev/null; then |
| KUBE_RESOURCE_FOUND="Firewall rules for ${MASTER_NAME}-all" |
| return 1 |
| fi |
| |
| local -a routes |
| kube::util::read-array routes < <(gcloud compute routes list --project "${NETWORK_PROJECT}" \ |
| --filter="name ~ '${INSTANCE_PREFIX}-minion-.{4}'" --format='value(name)') |
| if (( "${#routes[@]}" > 0 )); then |
| KUBE_RESOURCE_FOUND="${#routes[@]} routes matching ${INSTANCE_PREFIX}-minion-.{4}" |
| return 1 |
| fi |
| |
| if gcloud compute addresses describe --project "${PROJECT}" "${MASTER_NAME}-ip" --region "${REGION}" &>/dev/null; then |
| KUBE_RESOURCE_FOUND="Master's reserved IP" |
| return 1 |
| fi |
| |
| # No resources found. |
| return 0 |
| } |
| |
| # ----------------------------------------------------------------------------- |
| # Cluster specific test helpers |
| |
| # Execute prior to running tests to build a release if required for env. |
| # |
| # Assumed Vars: |
| # KUBE_ROOT |
| function test-build-release() { |
| # Make a release |
| "${KUBE_ROOT}/build/release.sh" |
| } |
| |
| # Execute prior to running tests to initialize required structure. |
| # |
| # Assumed vars: |
| # Variables from config.sh |
| function test-setup() { |
| # Detect the project into $PROJECT if it isn't set |
| detect-project |
| |
| if [[ ${MULTIZONE:-} == "true" && -n ${E2E_ZONES:-} ]]; then |
| for KUBE_GCE_ZONE in ${E2E_ZONES}; do |
| KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_USE_EXISTING_MASTER="${KUBE_USE_EXISTING_MASTER:-}" "${KUBE_ROOT}/cluster/kube-up.sh" |
| KUBE_USE_EXISTING_MASTER="true" # For subsequent zones we use the existing master |
| done |
| else |
| "${KUBE_ROOT}/cluster/kube-up.sh" |
| fi |
| |
| # Open up port 80 & 8080 so common containers on minions can be reached |
| # TODO(roberthbailey): Remove this once we are no longer relying on hostPorts. |
| local start |
| start=$(date +%s) |
| gcloud compute firewall-rules create \ |
| --project "${NETWORK_PROJECT}" \ |
| --target-tags "${NODE_TAG}" \ |
| --allow tcp:80,tcp:8080 \ |
| --network "${NETWORK}" \ |
| "${NODE_TAG}-http-alt" 2> /dev/null || true |
| # As there is no simple way to wait longer for this operation we need to manually |
| # wait some additional time (20 minutes altogether). |
| while ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NODE_TAG}-http-alt" 2> /dev/null; do |
| if [[ $((start + 1200)) -lt $(date +%s) ]]; then |
| echo -e "${color_red:-}Failed to create firewall ${NODE_TAG}-http-alt in ${NETWORK_PROJECT}" >&2 |
| exit 1 |
| fi |
| sleep 5 |
| done |
| |
| # Open up the NodePort range |
| # TODO(justinsb): Move to main setup, if we decide whether we want to do this by default. |
| start=$(date +%s) |
| gcloud compute firewall-rules create \ |
| --project "${NETWORK_PROJECT}" \ |
| --target-tags "${NODE_TAG}" \ |
| --allow tcp:30000-32767,udp:30000-32767 \ |
| --network "${NETWORK}" \ |
| "${NODE_TAG}-nodeports" 2> /dev/null || true |
| # As there is no simple way to wait longer for this operation we need to manually |
| # wait some additional time (20 minutes altogether). |
| while ! gcloud compute firewall-rules describe --project "${NETWORK_PROJECT}" "${NODE_TAG}-nodeports" 2> /dev/null; do |
| if [[ $((start + 1200)) -lt $(date +%s) ]]; then |
| echo -e "${color_red}Failed to create firewall ${NODE_TAG}-nodeports in ${PROJECT}" >&2 |
| exit 1 |
| fi |
| sleep 5 |
| done |
| } |
| |
| # Execute after running tests to perform any required clean-up. |
| function test-teardown() { |
| detect-project |
| echo "Shutting down test cluster in background." |
| delete-firewall-rules \ |
| "${NODE_TAG}-http-alt" \ |
| "${NODE_TAG}-nodeports" |
| if [[ ${MULTIZONE:-} == "true" && -n ${E2E_ZONES:-} ]]; then |
| local zones |
| read -r -a zones <<< "${E2E_ZONES}" |
| # tear them down in reverse order, finally tearing down the master too. |
| for ((zone_num=${#zones[@]}-1; zone_num>0; zone_num--)); do |
| KUBE_GCE_ZONE="${zones[zone_num]}" KUBE_USE_EXISTING_MASTER="true" "${KUBE_ROOT}/cluster/kube-down.sh" |
| done |
| KUBE_GCE_ZONE="${zones[0]}" KUBE_USE_EXISTING_MASTER="false" "${KUBE_ROOT}/cluster/kube-down.sh" |
| else |
| "${KUBE_ROOT}/cluster/kube-down.sh" |
| fi |
| } |
| |
| # SSH to a node by name ($1) and run a command ($2). |
| function ssh-to-node() { |
| local node="$1" |
| local cmd="$2" |
| # Loop until we can successfully ssh into the box |
| for (( i=0; i<5; i++)); do |
| if gcloud compute ssh --ssh-flag='-o LogLevel=quiet' --ssh-flag='-o ConnectTimeout=30' --project "${PROJECT}" --zone="${ZONE}" "${node}" --command 'echo test > /dev/null'; then |
| break |
| fi |
| sleep 5 |
| done |
| # Then actually try the command. |
| gcloud compute ssh --ssh-flag="-o LogLevel=quiet" --ssh-flag="-o ConnectTimeout=30" --project "${PROJECT}" --zone="${ZONE}" "${node}" --command "${cmd}" |
| } |
| |
| # Perform preparations required to run e2e tests |
| function prepare-e2e() { |
| detect-project |
| } |
| |
| # Delete the image given by $1. |
| function delete-image() { |
| gcloud container images delete --quiet "$1" |
| } |
