From 8739d508bdc438138938e4e87418dcdd5fe9dcaf Mon Sep 17 00:00:00 2001 From: upodroid Date: Tue, 6 Oct 2026 00:30:23 +0100 Subject: [PATCH] delete kubemark --- build/dependencies.yaml | 2 - cluster/gce/manifests/kube-addon-manager.yaml | 2 - cluster/images/kubemark/Dockerfile | 24 - cluster/images/kubemark/Makefile | 36 - cluster/images/kubemark/OWNERS | 12 - cluster/kubemark/OWNERS | 14 - cluster/kubemark/gce/config-default.sh | 61 -- cluster/kubemark/iks/config-default.sh | 44 - .../kubemark/pre-existing/config-default.sh | 37 - cluster/kubemark/util.sh | 20 - cluster/log-dump/log-dump.sh | 8 - cluster/log-dump/logexporter-daemonset.yaml | 1 - cluster/pre-existing/OWNERS | 4 - cluster/pre-existing/util.sh | 45 - cmd/kubemark/OWNERS | 12 - cmd/kubemark/app/hollow_node.go | 301 ------- cmd/kubemark/app/hollow_node_test.go | 96 --- cmd/kubemark/hollow-node.go | 30 - hack/lib/golang.sh | 3 - pkg/.import-restrictions | 1 - pkg/kubemark/.import-restrictions | 5 - pkg/kubemark/OWNERS | 14 - pkg/kubemark/controller.go | 401 --------- pkg/kubemark/hollow_kubelet.go | 213 ----- pkg/proxy/kubemark/.import-restrictions | 6 - pkg/proxy/kubemark/OWNERS | 14 - pkg/proxy/kubemark/hollow_proxy.go | 93 --- test/e2e/apimachinery/garbage_collector.go | 2 +- test/e2e/framework/debug/init/init.go | 1 - .../debug/resource_usage_gatherer.go | 113 +-- test/e2e/framework/framework.go | 7 +- test/e2e/framework/metrics/grab.go | 6 +- test/e2e/framework/metrics/kubelet_metrics.go | 2 +- test/e2e/framework/metrics/metrics_grabber.go | 20 +- .../providers/kubemark/.import-restrictions | 9 - .../framework/providers/kubemark/kubemark.go | 97 --- test/e2e/framework/ssh/ssh.go | 2 +- test/e2e/instrumentation/metrics.go | 5 +- .../monitoring/metrics_grabber.go | 5 +- test/e2e/network/kube_proxy.go | 2 +- test/e2e/providers.go | 1 - test/e2e/storage/csimock/csi_selinux_mount.go | 2 +- test/e2e/storage/csimock/csi_snapshot.go | 2 +- test/e2e/storage/testsuites/base.go | 2 +- test/e2e/storage/volume_metrics.go | 2 +- test/e2e/suites.go | 4 +- test/kubemark/OWNERS | 13 - test/kubemark/cloud-provider-config.sh | 21 - test/kubemark/common/util.sh | 49 -- test/kubemark/configure-kubectl.sh | 20 - test/kubemark/gce/util.sh | 137 ---- test/kubemark/pre-existing/README.md | 54 -- test/kubemark/pre-existing/util.sh | 27 - .../cluster-autoscaler_template.json | 94 --- .../kubemark/resources/heapster_template.json | 82 -- .../resources/hollow-node_template.yaml | 138 ---- test/kubemark/resources/kernel-monitor.json | 20 - .../kubemark/resources/kube_dns_template.yaml | 187 ----- test/kubemark/resources/kubemark-ns.json | 7 - .../addons/kubemark-rbac-bindings/README.md | 1 - .../cluster-autoscaler-binding.yaml | 16 - .../heapster-binding.yaml | 15 - .../kube-dns-binding.yaml | 15 - .../kubecfg-binding.yaml | 18 - .../kubelet-binding.yaml | 18 - .../kubemark-rbac-bindings/npd-binding.yaml | 15 - .../manifests/kube-addon-manager.yaml | 43 - .../resources/start-kubemark-master.sh | 776 ------------------ test/kubemark/run-e2e-tests.sh | 44 - test/kubemark/skeleton/util.sh | 62 -- test/kubemark/start-kubemark.sh | 255 ------ test/kubemark/stop-kubemark.sh | 46 -- 72 files changed, 32 insertions(+), 3924 deletions(-) delete mode 100644 cluster/images/kubemark/Dockerfile delete mode 100644 cluster/images/kubemark/Makefile delete mode 100644 cluster/images/kubemark/OWNERS delete mode 100644 cluster/kubemark/OWNERS delete mode 100644 cluster/kubemark/gce/config-default.sh delete mode 100644 cluster/kubemark/iks/config-default.sh delete mode 100644 cluster/kubemark/pre-existing/config-default.sh delete mode 100644 cluster/kubemark/util.sh delete mode 100644 cluster/pre-existing/OWNERS delete mode 100644 cluster/pre-existing/util.sh delete mode 100644 cmd/kubemark/OWNERS delete mode 100644 cmd/kubemark/app/hollow_node.go delete mode 100644 cmd/kubemark/app/hollow_node_test.go delete mode 100644 cmd/kubemark/hollow-node.go delete mode 100644 pkg/kubemark/.import-restrictions delete mode 100644 pkg/kubemark/OWNERS delete mode 100644 pkg/kubemark/controller.go delete mode 100644 pkg/kubemark/hollow_kubelet.go delete mode 100644 pkg/proxy/kubemark/.import-restrictions delete mode 100644 pkg/proxy/kubemark/OWNERS delete mode 100644 pkg/proxy/kubemark/hollow_proxy.go delete mode 100644 test/e2e/framework/providers/kubemark/.import-restrictions delete mode 100644 test/e2e/framework/providers/kubemark/kubemark.go delete mode 100644 test/kubemark/OWNERS delete mode 100755 test/kubemark/cloud-provider-config.sh delete mode 100644 test/kubemark/common/util.sh delete mode 100755 test/kubemark/configure-kubectl.sh delete mode 100644 test/kubemark/gce/util.sh delete mode 100644 test/kubemark/pre-existing/README.md delete mode 100644 test/kubemark/pre-existing/util.sh delete mode 100644 test/kubemark/resources/cluster-autoscaler_template.json delete mode 100644 test/kubemark/resources/heapster_template.json delete mode 100644 test/kubemark/resources/hollow-node_template.yaml delete mode 100644 test/kubemark/resources/kernel-monitor.json delete mode 100644 test/kubemark/resources/kube_dns_template.yaml delete mode 100644 test/kubemark/resources/kubemark-ns.json delete mode 100644 test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/README.md delete mode 100644 test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/cluster-autoscaler-binding.yaml delete mode 100644 test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/heapster-binding.yaml delete mode 100644 test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kube-dns-binding.yaml delete mode 100644 test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kubecfg-binding.yaml delete mode 100644 test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kubelet-binding.yaml delete mode 100644 test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/npd-binding.yaml delete mode 100644 test/kubemark/resources/manifests/kube-addon-manager.yaml delete mode 100755 test/kubemark/resources/start-kubemark-master.sh delete mode 100755 test/kubemark/run-e2e-tests.sh delete mode 100644 test/kubemark/skeleton/util.sh delete mode 100755 test/kubemark/start-kubemark.sh delete mode 100755 test/kubemark/stop-kubemark.sh diff --git a/build/dependencies.yaml b/build/dependencies.yaml index 76552e88aad..1f98935738d 100644 --- a/build/dependencies.yaml +++ b/build/dependencies.yaml @@ -122,8 +122,6 @@ dependencies: refPaths: - path: test/e2e_node/image_list.go match: const defaultImage - - path: test/kubemark/resources/hollow-node_template.yaml - match: registry.k8s.io/node-problem-detector/node-problem-detector #- path: cluster/gce/windows/k8s-node-setup.psm1 # match: DEFAULT_NPD_VERSION diff --git a/cluster/gce/manifests/kube-addon-manager.yaml b/cluster/gce/manifests/kube-addon-manager.yaml index ed17b592825..1cd2cad541b 100644 --- a/cluster/gce/manifests/kube-addon-manager.yaml +++ b/cluster/gce/manifests/kube-addon-manager.yaml @@ -21,8 +21,6 @@ spec: capabilities: drop: - all - # When updating version also bump it in: - # - test/kubemark/resources/manifests/kube-addon-manager.yaml image: registry.k8s.io/addon-manager/kube-addon-manager:v9.1.8 command: - /bin/bash diff --git a/cluster/images/kubemark/Dockerfile b/cluster/images/kubemark/Dockerfile deleted file mode 100644 index 229116d32a6..00000000000 --- a/cluster/images/kubemark/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# The line below points to the latest go-runner image, a wrapper around -# gcr.io/distroless/static which adds the go-runner command that we need -# for redirecting log output. -# -# See https://console.cloud.google.com/gcr/images/k8s-staging-build-image/global/go-runner -# for a list of available versions. This base image should be updated -# periodically. -FROM registry.k8s.io/build-image/go-runner:v2.3.1-go1.17.2-bullseye.0 - -COPY kubemark /kubemark diff --git a/cluster/images/kubemark/Makefile b/cluster/images/kubemark/Makefile deleted file mode 100644 index 611e1712cdb..00000000000 --- a/cluster/images/kubemark/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# build Kubemark image from currently built binaries containing both 'real' master and Hollow Node. -# This makefile assumes that the kubemark binary is present in this directory. - -# Allow the caller to override this. Beware make's precedence. This: -# REGISTRY=$VAR make -# .. is not the same as: -# make REGISTRY=$VAR -REGISTRY := $(if $(REGISTRY),$(REGISTRY),staging-k8s.gcr.io) -IMAGE_TAG := $(if $(IMAGE_TAG),$(IMAGE_TAG),latest) - -all: gcloudpush - -build: - docker build --pull -t $(REGISTRY)/kubemark:$(IMAGE_TAG) . - -gcloudpush: build - docker push $(REGISTRY)/kubemark:$(IMAGE_TAG) - -push: build - docker push $(REGISTRY)/kubemark:$(IMAGE_TAG) - -.PHONY: all build gcloudpush push diff --git a/cluster/images/kubemark/OWNERS b/cluster/images/kubemark/OWNERS deleted file mode 100644 index e167127f1db..00000000000 --- a/cluster/images/kubemark/OWNERS +++ /dev/null @@ -1,12 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: - - shyamjvs - - wojtek-t -approvers: - - shyamjvs - - wojtek-t -emeritus_approvers: - - gmarek -labels: - - sig/scalability diff --git a/cluster/kubemark/OWNERS b/cluster/kubemark/OWNERS deleted file mode 100644 index 16eb9516d79..00000000000 --- a/cluster/kubemark/OWNERS +++ /dev/null @@ -1,14 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: - - shyamjvs - - sig-scalability-reviewers - - wojtek-t -approvers: - - shyamjvs - - sig-scalability-approvers - - wojtek-t -emeritus_approvers: - - gmarek -labels: - - sig/scalability diff --git a/cluster/kubemark/gce/config-default.sh b/cluster/kubemark/gce/config-default.sh deleted file mode 100644 index dc2bc027ed7..00000000000 --- a/cluster/kubemark/gce/config-default.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A configuration for Kubemark cluster. It doesn't need to be kept in -# sync with gce/config-default.sh (except the filename, because I'm reusing -# gce/util.sh script which assumes config filename), but if some things that -# are enabled by default should not run in hollow clusters, they should be disabled here. - -# shellcheck disable=SC2034 # Variables sourced in other scripts. - -source "${KUBE_ROOT}/cluster/gce/config-common.sh" - -CLEANUP_KUBEMARK_IMAGE=${CLEANUP_KUBEMARK_IMAGE:-true} -TEST_CLUSTER_LOG_LEVEL="${TEST_CLUSTER_LOG_LEVEL:-"--v=4"}" - -# If you want to set up multiple kubemark clusters with different "names", -# you should change this env per each start-kubemark.sh invocation. -KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX:-"default"}" - -# NUM_NODES is used by start-kubemark.sh to determine a correct number of replicas. -NUM_NODES=${KUBEMARK_NUM_NODES:-10} -NUM_WINDOWS_NODES=${KUBEMARK_NUM_WINDOWS_NODES:-0} - -HOLLOW_KUBELET_TEST_LOG_LEVEL="${HOLLOW_KUBELET_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" -HOLLOW_PROXY_TEST_LOG_LEVEL="${HOLLOW_PROXY_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}" - -# Hollow-node components' test arguments. -HOLLOW_KUBELET_TEST_ARGS="${HOLLOW_KUBELET_TEST_ARGS:-} ${HOLLOW_KUBELET_TEST_LOG_LEVEL}" -HOLLOW_PROXY_TEST_ARGS="${HOLLOW_PROXY_TEST_ARGS:-} ${HOLLOW_PROXY_TEST_LOG_LEVEL}" - -# Optional: Enable cluster autoscaler. -ENABLE_KUBEMARK_CLUSTER_AUTOSCALER="${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-false}" -# When using Cluster Autoscaler, always start with one hollow-node replica. -# NUM_NODES should not be specified by the user. Instead we use -# NUM_NODES=KUBEMARK_AUTOSCALER_MAX_NODES. This gives other cluster components -# (e.g. kubemark master, Heapster) enough resources to handle maximum cluster size. -if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER}" == "true" ]]; then - NUM_REPLICAS=1 - if [[ -n "$NUM_NODES" ]]; then - echo "WARNING: Using Cluster Autoscaler, ignoring NUM_NODES parameter. Set KUBEMARK_AUTOSCALER_MAX_NODES to specify maximum size of the cluster." - fi -fi - -#Optional: Enable kube dns. -ENABLE_KUBEMARK_KUBE_DNS="${ENABLE_KUBEMARK_KUBE_DNS:-true}" -KUBE_DNS_DOMAIN="${KUBE_DNS_DOMAIN:-cluster.local}" - -CLEANUP_KUBEMARK_IMAGE=false diff --git a/cluster/kubemark/iks/config-default.sh b/cluster/kubemark/iks/config-default.sh deleted file mode 100644 index 779dee1e9df..00000000000 --- a/cluster/kubemark/iks/config-default.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2018 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Cloud information -RANDGEN=$(dd if=/dev/urandom bs=64 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=16 count=1 2>/dev/null | sed 's/[A-Z]//g') -# shellcheck disable=2034 # Variable sourced in other scripts. -KUBE_NAMESPACE="kubemark_${RANDGEN}" -KUBEMARK_IMAGE_TAG="${KUBEMARK_IMAGE_TAG:-2}" -KUBEMARK_IMAGE_LOCATION="${KUBEMARK_IMAGE_LOCATION:-${KUBE_ROOT}/cluster/images/kubemark}" -KUBEMARK_INIT_TAG="${KUBEMARK_INIT_TAG:-${PROJECT}:${KUBEMARK_IMAGE_TAG}}" -CLUSTER_LOCATION="${CLUSTER_LOCATION:-wdc06}" -REGISTRY_LOGIN_URL="${REGISTRY_LOGIN_URL:-https://api.ng.bluemix.net}" - -# User defined -# number of real workers in spawnTester cluster -NUM_NODES="${NUM_NODES:-2}" -# spec of real workers in spawnTester cluster -NODE_SIZE=${NODE_SIZE:-u2c.2x4} -DESIRED_NODES="${DESIRED_NODES:-10}" -# number of hollow nodes -# TODO: once IKS supports `kubetest`, i.e. the base cluster provisioning implemented in `kubetest` -# we can use NUM_NODES=${KUBEMARK_NUM_NODES:-10} to avoid usage of KUBEMARK_NUM_NODES -# more context, see https://github.com/kubernetes/kubernetes/pull/76909#discussion_r277735942 -KUBEMARK_NUM_NODES="${KUBEMARK_NUM_NODES:-10}" -ENABLE_KUBEMARK_CLUSTER_AUTOSCALER="${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-true}" -ENABLE_KUBEMARK_KUBE_DNS="${ENABLE_KUBEMARK_KUBE_DNS:-false}" -KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-"--v=2"}" -KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-"--v=4"}" -USE_REAL_PROXIER=${USE_REAL_PROXIER:-false} -NODE_INSTANCE_PREFIX=${NODE_INSTANCE_PREFIX:-node} -USE_EXISTING=${USE_EXISTING:-} diff --git a/cluster/kubemark/pre-existing/config-default.sh b/cluster/kubemark/pre-existing/config-default.sh deleted file mode 100644 index df5ae84c401..00000000000 --- a/cluster/kubemark/pre-existing/config-default.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Configuration for landing a Kubemark cluster on a pre-existing Kubernetes -# cluster. - -# Pre-existing provider expects a MASTER_IP. -# If you need to specify a port that's not the default (443), add it to MASTER_IP. -# -# Example: Connect to the Master on the secure port 6443 -# MASTER_IP=192.168.122.5:6443 -# -MASTER_IP="${MASTER_IP:-}" - -# The container registry and project given to the kubemark container: -# $CONTAINER_REGISTRY/$PROJECT/kubemark -# -CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-}" -PROJECT="${PROJECT:-}" - -NUM_NODES="${NUM_NODES:-1}" - -KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-}" -KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-}" -MASTER_NAME="${MASTER_NAME:-}" diff --git a/cluster/kubemark/util.sh b/cluster/kubemark/util.sh deleted file mode 100644 index 52ecbdf5daf..00000000000 --- a/cluster/kubemark/util.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2014 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. -source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh" -source "${KUBE_ROOT}/cluster/${CLOUD_PROVIDER}/util.sh" -source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh" diff --git a/cluster/log-dump/log-dump.sh b/cluster/log-dump/log-dump.sh index f73a9409e0d..b8ffbd82177 100755 --- a/cluster/log-dump/log-dump.sh +++ b/cluster/log-dump/log-dump.sh @@ -44,7 +44,6 @@ readonly gcloud_supported_providers="gce gke" readonly master_logfiles="kube-apiserver.log kube-apiserver-audit.log kube-scheduler.log kube-controller-manager.log cloud-controller-manager.log etcd.log etcd-events.log glbc.log cluster-autoscaler.log kube-addon-manager.log konnectivity-server.log fluentd.log kubelet.cov" readonly node_logfiles="kube-proxy.log containers/konnectivity-agent-*.log fluentd.log kubelet.cov kube-network-policies.log" readonly node_systemd_services="" -readonly hollow_node_logfiles="kubelet-hollow-node-*.log kubeproxy-hollow-node-*.log npd-hollow-node-*.log" readonly aws_logfiles="cloud-init-output.log" readonly gce_logfiles="startupscript.log" readonly kern_logfile="kern.log" @@ -375,8 +374,6 @@ function dump_masters() { elif [[ ! "${master_ssh_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then echo "Master SSH not supported for ${KUBERNETES_PROVIDER}" return - elif [[ -n "${KUBEMARK_MASTER_NAME:-}" ]]; then - master_names=( "${KUBEMARK_MASTER_NAME}" ) else if ! (detect-master); then echo 'Master not detected. Is the cluster up?' @@ -444,9 +441,6 @@ function dump_nodes() { fi node_logfiles_all="${node_logfiles}" - if [[ "${ENABLE_HOLLOW_NODE_LOGS:-}" == "true" ]]; then - node_logfiles_all="${node_logfiles_all} ${hollow_node_logfiles}" - fi linux_nodes_selected_for_logs=() if [[ -n "${LOGDUMP_ONLY_N_RANDOM_NODES:-}" ]]; then @@ -543,7 +537,6 @@ function dump_nodes_with_logexporter() { # Obtain parameters required by logexporter. local -r service_account_credentials="$(base64 "${GOOGLE_APPLICATION_CREDENTIALS}" | tr -d '\n')" local -r cloud_provider="${KUBERNETES_PROVIDER}" - local -r enable_hollow_node_logs="${ENABLE_HOLLOW_NODE_LOGS:-false}" local -r logexport_sleep_seconds="$(( 90 + NUM_NODES / 3 ))" if [[ -z "${ZONE_NODE_SELECTOR_DISABLED:-}" ]]; then local -r node_selector="${ZONE_NODE_SELECTOR_LABEL:-topology.kubernetes.io/zone}: ${ZONE}" @@ -560,7 +553,6 @@ function dump_nodes_with_logexporter() { sed -i'' -e "s@{{.ServiceAccountCredentials}}@${service_account_credentials}@g" "${manifest_yaml}" sed -i'' -e "s@{{.CloudProvider}}@${cloud_provider}@g" "${manifest_yaml}" sed -i'' -e "s@{{.GCSPath}}@${gcs_artifacts_dir}@g" "${manifest_yaml}" - sed -i'' -e "s@{{.EnableHollowNodeLogs}}@${enable_hollow_node_logs}@g" "${manifest_yaml}" sed -i'' -e "s@{{.DumpSystemdJournal}}@${dump_systemd_journal}@g" "${manifest_yaml}" sed -i'' -e "s@{{.ExtraLogFiles}}@${extra_log_files}@g" "${manifest_yaml}" sed -i'' -e "s@{{.ExtraSystemdServices}}@${extra_systemd_services}@g" "${manifest_yaml}" diff --git a/cluster/log-dump/logexporter-daemonset.yaml b/cluster/log-dump/logexporter-daemonset.yaml index 8279ae62f68..8c575ecc860 100644 --- a/cluster/log-dump/logexporter-daemonset.yaml +++ b/cluster/log-dump/logexporter-daemonset.yaml @@ -48,7 +48,6 @@ spec: - --cloud-provider={{.CloudProvider}} - --gcs-path={{.GCSPath}} - --gcloud-auth-file-path=/etc/service-account/service-account.json - - --enable-hollow-node-logs={{.EnableHollowNodeLogs}} - --dump-systemd-journal={{.DumpSystemdJournal}} - --extra-log-files={{.ExtraLogFiles}} - --extra-systemd-services={{.ExtraSystemdServices}} diff --git a/cluster/pre-existing/OWNERS b/cluster/pre-existing/OWNERS deleted file mode 100644 index 5d179b9203f..00000000000 --- a/cluster/pre-existing/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -labels: - - sig/scalability diff --git a/cluster/pre-existing/util.sh b/cluster/pre-existing/util.sh deleted file mode 100644 index 7dd3c632fb6..00000000000 --- a/cluster/pre-existing/util.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A library of helper functions for landing kubemark containers on a -# pre-existing Kubernetes master. See test/kubemark/pre-existing/README.md -# for me details on using a pre-existing provider. - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. - -source "${KUBE_ROOT}/cluster/common.sh" -source "${KUBE_ROOT}/hack/lib/util.sh" - -function detect-project() { - if [[ -z "${MASTER_IP:-}" ]]; then - echo "Set 'MASTER_IP' to the instance assigned to be the Kubernetes master" 1>&2 - exit 1 - fi - - if [[ -z "${PROJECT:-}" ]]; then - echo "Set 'PROJECT' to the name of the container project: $CONTAINER_REGISTRY/$PROJECT/kubemark" >&2 - exit 1 - fi - - if [[ -z "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then - cluster_range=$(echo "${MASTER_IP}" | awk -F '.' '{printf("%d.%d.%d.0", $1, $2, $3)}') - SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-$cluster_range/16}" - fi -} - -function create-certs { - execute-cmd-on-pre-existing-master-with-retries 'sudo cat /etc/kubernetes/admin.conf' > /tmp/kubeconfig -} diff --git a/cmd/kubemark/OWNERS b/cmd/kubemark/OWNERS deleted file mode 100644 index b3abbf8fbbf..00000000000 --- a/cmd/kubemark/OWNERS +++ /dev/null @@ -1,12 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: - - shyamjvs - - sig-scalability-reviewers - - wojtek-t -approvers: - - shyamjvs - - sig-scalability-approvers - - wojtek-t -emeritus_approvers: - - gmarek diff --git a/cmd/kubemark/app/hollow_node.go b/cmd/kubemark/app/hollow_node.go deleted file mode 100644 index 0d46c2a5cbf..00000000000 --- a/cmd/kubemark/app/hollow_node.go +++ /dev/null @@ -1,301 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "context" - goflag "flag" - "fmt" - "time" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" - internalapi "k8s.io/cri-api/pkg/apis" - "k8s.io/klog/v2" - - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" - clientset "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/client-go/tools/events" - cliflag "k8s.io/component-base/cli/flag" - _ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration - _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration - "k8s.io/component-base/version" - "k8s.io/component-base/version/verflag" - remote "k8s.io/cri-client/pkg" - fakeremote "k8s.io/cri-client/pkg/fake" - "k8s.io/kubernetes/pkg/api/legacyscheme" - "k8s.io/kubernetes/pkg/cluster/ports" - cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" - "k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap" - "k8s.io/kubernetes/pkg/kubelet/cm" - "k8s.io/kubernetes/pkg/kubemark" - kubemarkproxy "k8s.io/kubernetes/pkg/proxy/kubemark" - utilflag "k8s.io/kubernetes/pkg/util/flag" -) - -type hollowNodeConfig struct { - KubeconfigPath string - BootstrapKubeconfigPath string - CertDirectory string - KubeletPort int - KubeletReadOnlyPort int - Morph string - NodeName string - ServerPort int - ContentType string - QPS float32 - Burst int - NodeLabels map[string]string - RegisterWithTaints []v1.Taint - MaxPods int - ExtendedResources map[string]string - UseHostImageService bool - - // Deprecated config; remove these with the corresponding flags - UseRealProxier bool - ProxierSyncPeriod time.Duration - ProxierMinSyncPeriod time.Duration -} - -const ( - maxPods = 110 - podsPerCore = 0 -) - -// TODO(#45650): Refactor hollow-node into hollow-kubelet and hollow-proxy -// and make the config driven. -var knownMorphs = sets.NewString("kubelet", "proxy") - -func (c *hollowNodeConfig) addFlags(fs *pflag.FlagSet) { - fs.StringVar(&c.KubeconfigPath, "kubeconfig", "/kubeconfig/kubeconfig", "Path to kubeconfig file.") - fs.StringVar(&c.BootstrapKubeconfigPath, "bootstrap-kubeconfig", "", "Path to bootstrap kubeconfig file.") - fs.StringVar(&c.CertDirectory, "cert-dir", "/etc/srv/", "Path to cert directory for bootstraping.") - fs.IntVar(&c.KubeletPort, "kubelet-port", ports.KubeletPort, "Port on which HollowKubelet should be listening.") - fs.IntVar(&c.KubeletReadOnlyPort, "kubelet-read-only-port", ports.KubeletReadOnlyPort, "Read-only port on which Kubelet is listening.") - fs.StringVar(&c.NodeName, "name", "fake-node", "Name of this Hollow Node.") - fs.IntVar(&c.ServerPort, "api-server-port", 443, "Port on which API server is listening.") - fs.StringVar(&c.Morph, "morph", "", fmt.Sprintf("Specifies into which Hollow component this binary should morph. Allowed values: %v", knownMorphs.List())) - fs.StringVar(&c.ContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType of requests sent to apiserver.") - fs.Float32Var(&c.QPS, "kube-api-qps", 10, "QPS indicates the maximum QPS to the apiserver.") - fs.IntVar(&c.Burst, "kube-api-burst", 20, "Burst indicates maximum burst for throttle to the apiserver.") - - bindableNodeLabels := cliflag.ConfigurationMap(c.NodeLabels) - fs.Var(&bindableNodeLabels, "node-labels", "Additional node labels") - fs.Var(utilflag.RegisterWithTaintsVar{Value: &c.RegisterWithTaints}, "register-with-taints", "Register the node with the given list of taints (comma separated \"=:\"). No-op if register-node is false.") - fs.IntVar(&c.MaxPods, "max-pods", maxPods, "Number of pods that can run on this Kubelet.") - bindableExtendedResources := cliflag.ConfigurationMap(c.ExtendedResources) - fs.Var(&bindableExtendedResources, "extended-resources", "Register the node with extended resources (comma separated \"=\")") - fs.BoolVar(&c.UseHostImageService, "use-host-image-service", true, "Set to true if the hollow-kubelet should use the host image service. If set to false the fake image service will be used") - - fs.BoolVar(&c.UseRealProxier, "use-real-proxier", true, "Has no effect.") - _ = fs.MarkDeprecated("use-real-proxier", "This flag is deprecated and will be removed in a future release.") - fs.DurationVar(&c.ProxierSyncPeriod, "proxier-sync-period", 30*time.Second, "Has no effect.") - _ = fs.MarkDeprecated("proxier-sync-period", "This flag is deprecated and will be removed in a future release.") - fs.DurationVar(&c.ProxierMinSyncPeriod, "proxier-min-sync-period", 0, "Has no effect.") - _ = fs.MarkDeprecated("proxier-min-sync-period", "This flag is deprecated and will be removed in a future release.") -} - -func (c *hollowNodeConfig) createClientConfigFromFile() (*restclient.Config, error) { - clientConfig, err := clientcmd.LoadFromFile(c.KubeconfigPath) - if err != nil { - return nil, fmt.Errorf("error while loading kubeconfig from file %v: %v", c.KubeconfigPath, err) - } - config, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig() - if err != nil { - return nil, fmt.Errorf("error while creating kubeconfig: %v", err) - } - config.ContentType = c.ContentType - config.QPS = c.QPS - config.Burst = c.Burst - return config, nil -} - -func (c *hollowNodeConfig) bootstrapClientConfig() error { - if c.BootstrapKubeconfigPath != "" { - return bootstrap.LoadClientCert(context.TODO(), c.KubeconfigPath, c.BootstrapKubeconfigPath, c.CertDirectory, types.NodeName(c.NodeName)) - } - return nil -} - -func (c *hollowNodeConfig) createHollowKubeletOptions() *kubemark.HollowKubeletOptions { - return &kubemark.HollowKubeletOptions{ - NodeName: c.NodeName, - KubeletPort: c.KubeletPort, - KubeletReadOnlyPort: c.KubeletReadOnlyPort, - MaxPods: c.MaxPods, - PodsPerCore: podsPerCore, - NodeLabels: c.NodeLabels, - RegisterWithTaints: c.RegisterWithTaints, - } -} - -// NewHollowNodeCommand creates a *cobra.Command object with default parameters -func NewHollowNodeCommand() *cobra.Command { - s := &hollowNodeConfig{ - NodeLabels: make(map[string]string), - ExtendedResources: make(map[string]string), - } - - cmd := &cobra.Command{ - Use: "kubemark", - Long: "kubemark", - RunE: func(cmd *cobra.Command, args []string) error { - verflag.PrintAndExitIfRequested() - cliflag.PrintFlags(cmd.Flags()) - return run(cmd.Context(), s) - }, - Args: func(cmd *cobra.Command, args []string) error { - for _, arg := range args { - if len(arg) > 0 { - return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args) - } - } - return nil - }, - } - - fs := cmd.Flags() - fs.AddGoFlagSet(goflag.CommandLine) // for flags like --docker-only - s.addFlags(fs) - - return cmd -} - -func run(ctx context.Context, config *hollowNodeConfig) error { - // To help debugging, immediately log version and print flags. - klog.Infof("Version: %+v", version.Get()) - - if !knownMorphs.Has(config.Morph) { - return fmt.Errorf("Unknown morph: %v. allowed values: %v", config.Morph, knownMorphs.List()) - } - - // create a client to communicate with API server. - err := config.bootstrapClientConfig() - if err != nil { - return fmt.Errorf("Failed to bootstrap, error: %w. Exiting", err) - } - clientConfig, err := config.createClientConfigFromFile() - if err != nil { - return fmt.Errorf("Failed to create a ClientConfig, error: %w. Exiting", err) - } - - if config.Morph == "kubelet" { - clientConfig.UserAgent = "hollow-kubelet" - client, err := clientset.NewForConfig(clientConfig) - if err != nil { - return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err) - } - - f, c := kubemark.GetHollowKubeletConfig(config.createHollowKubeletOptions()) - - heartbeatClientConfig := *clientConfig - heartbeatClientConfig.Timeout = c.NodeStatusUpdateFrequency.Duration - // The timeout is the minimum of the lease duration and status update frequency - leaseTimeout := time.Duration(c.NodeLeaseDurationSeconds) * time.Second - if heartbeatClientConfig.Timeout > leaseTimeout { - heartbeatClientConfig.Timeout = leaseTimeout - } - - heartbeatClientConfig.QPS = float32(-1) - heartbeatClient, err := clientset.NewForConfig(&heartbeatClientConfig) - if err != nil { - return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err) - } - - cadvisorInterface := &cadvisortest.Fake{ - NodeName: config.NodeName, - } - - var containerManager cm.ContainerManager - if config.ExtendedResources != nil { - extendedResources := v1.ResourceList{} - for k, v := range config.ExtendedResources { - extendedResources[v1.ResourceName(k)] = resource.MustParse(v) - } - - containerManager = cm.NewStubContainerManagerWithDevicePluginResource(extendedResources) - } else { - containerManager = cm.NewStubContainerManager() - } - - endpoint, err := fakeremote.GenerateEndpoint() - if err != nil { - return fmt.Errorf("Failed to generate fake endpoint, error: %w", err) - } - fakeRemoteRuntime := fakeremote.NewFakeRemoteRuntime() - if err = fakeRemoteRuntime.Start(endpoint); err != nil { - return fmt.Errorf("Failed to start fake runtime, error: %w", err) - } - defer fakeRemoteRuntime.Stop() - runtimeService, err := remote.NewRemoteRuntimeServiceBuilder(). - WithEndpoint(endpoint). - WithConnectionTimeout(15 * time.Second). - Build(ctx) - if err != nil { - return fmt.Errorf("Failed to init runtime service, error: %w", err) - } - - var imageService internalapi.ImageManagerService = fakeRemoteRuntime.ImageService - if config.UseHostImageService { - imageService, err = remote.NewRemoteImageServiceBuilder(). - WithEndpoint(c.ImageServiceEndpoint). - WithConnectionTimeout(15 * time.Second). - Build(ctx) - if err != nil { - return fmt.Errorf("Failed to init image service, error: %w", err) - } - } - - hollowKubelet := kubemark.NewHollowKubelet( - f, c, - client, - heartbeatClient, - cadvisorInterface, - imageService, - runtimeService, - containerManager, - ) - hollowKubelet.Run(ctx) - } - - if config.Morph == "proxy" { - clientConfig.UserAgent = "hollow-proxy" - - client, err := clientset.NewForConfig(clientConfig) - if err != nil { - return fmt.Errorf("Failed to create API Server client, error: %w", err) - } - eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()}) - recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, "kube-proxy") - - hollowProxy := kubemarkproxy.NewHollowProxy( - config.NodeName, - client, - client.CoreV1(), - eventBroadcaster, - recorder, - ) - return hollowProxy.Run() - } - - return nil -} diff --git a/cmd/kubemark/app/hollow_node_test.go b/cmd/kubemark/app/hollow_node_test.go deleted file mode 100644 index 000fde25368..00000000000 --- a/cmd/kubemark/app/hollow_node_test.go +++ /dev/null @@ -1,96 +0,0 @@ -/* -Copyright 2023 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" - "time" - - "k8s.io/kubernetes/test/utils/ktesting" -) - -const fakeKubeconfig = ` -apiVersion: v1 -kind: Config -clusters: -- cluster: - insecure-skip-tls-verify: true - server: %s - name: default -contexts: -- context: - cluster: default - user: default - name: default -current-context: default -users: -- name: default - user: - username: config -` - -// TestHollowNode is a naive test that attempts to start hollow node and checks if it's not crashing. -// Such test is sufficient to detect e.g. missing kubelet dependencies that are not added in -// pkg/kubemark/hollow_kubelet.go. -func TestHollowNode(t *testing.T) { - // temp dir - tmpDir, err := os.MkdirTemp("", "hollow-node") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpDir) - - // https server - server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - w.WriteHeader(200) - w.Write([]byte(`ok`)) - })) - defer server.Close() - - kubeconfigPath := filepath.Join(tmpDir, "config.kubeconfig") - if err := os.WriteFile(kubeconfigPath, []byte(fmt.Sprintf(fakeKubeconfig, server.URL)), os.FileMode(0600)); err != nil { - t.Fatal(err) - } - - for morph := range knownMorphs { - t.Run(morph, func(t *testing.T) { - s := &hollowNodeConfig{ - KubeconfigPath: kubeconfigPath, - Morph: morph, - } - errCh := make(chan error) - go func() { - data, err := os.ReadFile(kubeconfigPath) - t.Logf("read %d, err=%v\n", len(data), err) - ctx := ktesting.Init(t) - errCh <- run(ctx, s) - }() - - select { - case err := <-errCh: - t.Fatalf("Run finished unexpectedly with error: %v", err) - case <-time.After(3 * time.Second): - t.Logf("Morph %q hasn't crashed for 3s. Calling success.", morph) - } - }) - } -} diff --git a/cmd/kubemark/hollow-node.go b/cmd/kubemark/hollow-node.go deleted file mode 100644 index a80e11f70c1..00000000000 --- a/cmd/kubemark/hollow-node.go +++ /dev/null @@ -1,30 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "os" - - "k8s.io/component-base/cli" - "k8s.io/kubernetes/cmd/kubemark/app" -) - -func main() { - command := app.NewHollowNodeCommand() - code := cli.Run(command) - os.Exit(code) -} diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index 70ba7d0f6fe..7fc0e619e8d 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -277,7 +277,6 @@ readonly KUBE_TEST_BINARIES=("${KUBE_TEST_TARGETS[@]##*/}") readonly KUBE_TEST_BINARIES_WIN=("${KUBE_TEST_BINARIES[@]/%/.exe}") readonly KUBE_TEST_PORTABLE=( test/e2e/testing-manifests - test/kubemark hack/e2e-internal hack/get-build.sh hack/ginkgo-e2e.sh @@ -289,7 +288,6 @@ readonly KUBE_TEST_PORTABLE=( # These binaries will be distributed in the kubernetes-test tarball. kube::golang::server_test_targets() { local targets=( - cmd/kubemark ginkgo ) @@ -332,7 +330,6 @@ readonly KUBE_STATIC_BINARIES=( kubectl kubectl-convert kubelet - kubemark mounter ) diff --git a/pkg/.import-restrictions b/pkg/.import-restrictions index c030367ceeb..50ad00e3e83 100644 --- a/pkg/.import-restrictions +++ b/pkg/.import-restrictions @@ -4,7 +4,6 @@ rules: - '' forbiddenPrefixes: # prevent pkg/ from depending on cmd/ - # note: pkg/kubemark overrides this - k8s.io/kubernetes/cmd # use sigs.k8s.io/yaml instead - github.com/ghodss/yaml diff --git a/pkg/kubemark/.import-restrictions b/pkg/kubemark/.import-restrictions deleted file mode 100644 index 6390404bd0e..00000000000 --- a/pkg/kubemark/.import-restrictions +++ /dev/null @@ -1,5 +0,0 @@ -rules: - # override pkg/ import restriction on cmd/ for kubemark - - selectorRegexp: k8s[.]io/kubernetes/cmd - allowedPrefixes: - - k8s.io/kubernetes/cmd/kubelet/app diff --git a/pkg/kubemark/OWNERS b/pkg/kubemark/OWNERS deleted file mode 100644 index 16eb9516d79..00000000000 --- a/pkg/kubemark/OWNERS +++ /dev/null @@ -1,14 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: - - shyamjvs - - sig-scalability-reviewers - - wojtek-t -approvers: - - shyamjvs - - sig-scalability-approvers - - wojtek-t -emeritus_approvers: - - gmarek -labels: - - sig/scalability diff --git a/pkg/kubemark/controller.go b/pkg/kubemark/controller.go deleted file mode 100644 index fdb7c03eebb..00000000000 --- a/pkg/kubemark/controller.go +++ /dev/null @@ -1,401 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubemark - -import ( - "context" - "fmt" - "math/rand" - "sync" - "time" - - apiv1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/informers" - informersv1 "k8s.io/client-go/informers/core/v1" - kubeclient "k8s.io/client-go/kubernetes" - listersv1 "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/cache" - - "k8s.io/klog/v2" -) - -const ( - namespaceKubemark = "kubemark" - nodeGroupLabel = "autoscaling.k8s.io/nodegroup" - numRetries = 3 -) - -// KubemarkController is a simplified version of cloud provider for kubemark. It allows -// to add and delete nodes from a kubemark cluster and introduces nodegroups -// by applying labels to the kubemark's hollow-nodes. -type KubemarkController struct { - nodeTemplate *apiv1.ReplicationController - externalCluster externalCluster - kubemarkCluster kubemarkCluster - rand *rand.Rand - createNodeQueue chan string - nodeGroupQueueSize map[string]int - nodeGroupQueueSizeLock sync.Mutex -} - -// externalCluster is used to communicate with the external cluster that hosts -// kubemark, in order to be able to list, create and delete hollow nodes -// by manipulating the replication controllers. -type externalCluster struct { - rcLister listersv1.ReplicationControllerLister - rcSynced cache.InformerSynced - podLister listersv1.PodLister - podSynced cache.InformerSynced - client kubeclient.Interface -} - -// kubemarkCluster is used to delete nodes from kubemark cluster once their -// respective replication controllers have been deleted and the nodes have -// become unready. This is to cover for the fact that there is no proper cloud -// provider for kubemark that would care for deleting the nodes. -type kubemarkCluster struct { - client kubeclient.Interface - nodeLister listersv1.NodeLister - nodeSynced cache.InformerSynced - nodesToDelete map[string]bool - nodesToDeleteLock sync.Mutex -} - -// NewKubemarkController creates KubemarkController using the provided clients to talk to external -// and kubemark clusters. -func NewKubemarkController(externalClient kubeclient.Interface, externalInformerFactory informers.SharedInformerFactory, - kubemarkClient kubeclient.Interface, kubemarkNodeInformer informersv1.NodeInformer) (*KubemarkController, error) { - rcInformer := externalInformerFactory.InformerFor(&apiv1.ReplicationController{}, newReplicationControllerInformer) - podInformer := externalInformerFactory.InformerFor(&apiv1.Pod{}, newPodInformer) - controller := &KubemarkController{ - externalCluster: externalCluster{ - rcLister: listersv1.NewReplicationControllerLister(rcInformer.GetIndexer()), - rcSynced: rcInformer.HasSynced, - podLister: listersv1.NewPodLister(podInformer.GetIndexer()), - podSynced: podInformer.HasSynced, - client: externalClient, - }, - kubemarkCluster: kubemarkCluster{ - nodeLister: kubemarkNodeInformer.Lister(), - nodeSynced: kubemarkNodeInformer.Informer().HasSynced, - client: kubemarkClient, - nodesToDelete: make(map[string]bool), - nodesToDeleteLock: sync.Mutex{}, - }, - rand: rand.New(rand.NewSource(time.Now().UnixNano())), - createNodeQueue: make(chan string, 1000), - nodeGroupQueueSize: make(map[string]int), - nodeGroupQueueSizeLock: sync.Mutex{}, - } - - kubemarkNodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - UpdateFunc: controller.kubemarkCluster.removeUnneededNodes, - }) - - return controller, nil -} - -// WaitForCacheSync waits until all caches in the controller are populated. -func (kubemarkController *KubemarkController) WaitForCacheSync(stopCh chan struct{}) bool { - return cache.WaitForNamedCacheSync("kubemark", stopCh, - kubemarkController.externalCluster.rcSynced, - kubemarkController.externalCluster.podSynced, - kubemarkController.kubemarkCluster.nodeSynced) -} - -// Run populates the node template needed for creation of kubemark nodes and -// starts the worker routine for creating new nodes. -func (kubemarkController *KubemarkController) Run(stopCh chan struct{}) { - nodeTemplate, err := kubemarkController.getNodeTemplate() - if err != nil { - klog.Fatalf("failed to get node template: %s", err) - } - kubemarkController.nodeTemplate = nodeTemplate - - go kubemarkController.runNodeCreation(stopCh) - <-stopCh -} - -// GetNodeNamesForNodeGroup returns list of the nodes in the node group. -func (kubemarkController *KubemarkController) GetNodeNamesForNodeGroup(nodeGroup string) ([]string, error) { - selector := labels.SelectorFromSet(labels.Set{nodeGroupLabel: nodeGroup}) - pods, err := kubemarkController.externalCluster.podLister.List(selector) - if err != nil { - return nil, err - } - result := make([]string, 0, len(pods)) - for _, pod := range pods { - result = append(result, pod.ObjectMeta.Name) - } - return result, nil -} - -// GetNodeGroupSize returns the current size for the node group as observed. -func (kubemarkController *KubemarkController) GetNodeGroupSize(nodeGroup string) (int, error) { - selector := labels.SelectorFromSet(labels.Set(map[string]string{nodeGroupLabel: nodeGroup})) - nodes, err := kubemarkController.externalCluster.rcLister.List(selector) - if err != nil { - return 0, err - } - return len(nodes), nil -} - -// GetNodeGroupTargetSize returns the size of the node group as a sum of current -// observed size and number of upcoming nodes. -func (kubemarkController *KubemarkController) GetNodeGroupTargetSize(nodeGroup string) (int, error) { - kubemarkController.nodeGroupQueueSizeLock.Lock() - defer kubemarkController.nodeGroupQueueSizeLock.Unlock() - realSize, err := kubemarkController.GetNodeGroupSize(nodeGroup) - if err != nil { - return realSize, err - } - return realSize + kubemarkController.nodeGroupQueueSize[nodeGroup], nil -} - -// SetNodeGroupSize changes the size of node group by adding or removing nodes. -func (kubemarkController *KubemarkController) SetNodeGroupSize(nodeGroup string, size int) error { - currSize, err := kubemarkController.GetNodeGroupTargetSize(nodeGroup) - if err != nil { - return err - } - switch delta := size - currSize; { - case delta < 0: - absDelta := -delta - nodes, err := kubemarkController.GetNodeNamesForNodeGroup(nodeGroup) - if err != nil { - return err - } - if len(nodes) < absDelta { - return fmt.Errorf("can't remove %d nodes from %s nodegroup, not enough nodes: %d", absDelta, nodeGroup, len(nodes)) - } - for i, node := range nodes { - if i == absDelta { - return nil - } - if err := kubemarkController.RemoveNodeFromNodeGroup(nodeGroup, node); err != nil { - return err - } - } - case delta > 0: - kubemarkController.nodeGroupQueueSizeLock.Lock() - kubemarkController.nodeGroupQueueSize[nodeGroup] += delta - kubemarkController.nodeGroupQueueSizeLock.Unlock() - for i := 0; i < delta; i++ { - kubemarkController.createNodeQueue <- nodeGroup - } - } - - return nil -} - -// GetNodeGroupForNode returns the name of the node group to which the node -// belongs. -func (kubemarkController *KubemarkController) GetNodeGroupForNode(node string) (string, error) { - pod := kubemarkController.getPodByName(node) - if pod == nil { - return "", fmt.Errorf("node %s does not exist", node) - } - nodeGroup, ok := pod.ObjectMeta.Labels[nodeGroupLabel] - if ok { - return nodeGroup, nil - } - return "", fmt.Errorf("can't find nodegroup for node %s due to missing label %s", node, nodeGroupLabel) -} - -func (kubemarkController *KubemarkController) addNodeToNodeGroup(nodeGroup string) error { - node := kubemarkController.nodeTemplate.DeepCopy() - node.Name = fmt.Sprintf("%s-%d", nodeGroup, kubemarkController.rand.Int63()) - node.Labels = map[string]string{nodeGroupLabel: nodeGroup, "name": node.Name} - node.Spec.Template.Labels = node.Labels - - var err error - for i := 0; i < numRetries; i++ { - _, err = kubemarkController.externalCluster.client.CoreV1().ReplicationControllers(node.Namespace).Create(context.TODO(), node, metav1.CreateOptions{}) - if err == nil { - return nil - } - } - - return err -} - -func (kubemarkController *KubemarkController) RemoveNodeFromNodeGroup(nodeGroup string, node string) error { - pod := kubemarkController.getPodByName(node) - if pod == nil { - klog.Warningf("Can't delete node %s from nodegroup %s. Node does not exist.", node, nodeGroup) - return nil - } - if pod.ObjectMeta.Labels[nodeGroupLabel] != nodeGroup { - return fmt.Errorf("can't delete node %s from nodegroup %s. Node is not in nodegroup", node, nodeGroup) - } - policy := metav1.DeletePropagationForeground - var err error - for i := 0; i < numRetries; i++ { - err = kubemarkController.externalCluster.client.CoreV1().ReplicationControllers(namespaceKubemark).Delete(context.TODO(), pod.ObjectMeta.Labels["name"], - metav1.DeleteOptions{PropagationPolicy: &policy}) - if err == nil { - klog.Infof("marking node %s for deletion", node) - // Mark node for deletion from kubemark cluster. - // Once it becomes unready after replication controller - // deletion has been noticed, we will delete it explicitly. - // This is to cover for the fact that kubemark does not - // take care of this itself. - kubemarkController.kubemarkCluster.markNodeForDeletion(node) - return nil - } - } - return fmt.Errorf("Failed to delete node %s: %v", node, err) -} - -func (kubemarkController *KubemarkController) getReplicationControllerByName(name string) *apiv1.ReplicationController { - rcs, err := kubemarkController.externalCluster.rcLister.List(labels.Everything()) - if err != nil { - return nil - } - for _, rc := range rcs { - if rc.ObjectMeta.Name == name { - return rc - } - } - return nil -} - -func (kubemarkController *KubemarkController) getPodByName(name string) *apiv1.Pod { - pods, err := kubemarkController.externalCluster.podLister.List(labels.Everything()) - if err != nil { - return nil - } - for _, pod := range pods { - if pod.ObjectMeta.Name == name { - return pod - } - } - return nil -} - -func (kubemarkController *KubemarkController) getNodeNameForPod(podName string) (string, error) { - pods, err := kubemarkController.externalCluster.podLister.List(labels.Everything()) - if err != nil { - return "", err - } - for _, pod := range pods { - if pod.ObjectMeta.Name == podName { - return pod.Labels["name"], nil - } - } - return "", fmt.Errorf("pod %s not found", podName) -} - -// getNodeTemplate returns the template for hollow node replication controllers -// by looking for an existing hollow node specification. This requires at least -// one kubemark node to be present on startup. -func (kubemarkController *KubemarkController) getNodeTemplate() (*apiv1.ReplicationController, error) { - podName, err := kubemarkController.kubemarkCluster.getHollowNodeName() - if err != nil { - return nil, err - } - hollowNodeName, err := kubemarkController.getNodeNameForPod(podName) - if err != nil { - return nil, err - } - if hollowNode := kubemarkController.getReplicationControllerByName(hollowNodeName); hollowNode != nil { - nodeTemplate := &apiv1.ReplicationController{ - Spec: apiv1.ReplicationControllerSpec{ - Template: hollowNode.Spec.Template, - }, - } - - nodeTemplate.Spec.Selector = nil - nodeTemplate.Namespace = namespaceKubemark - one := int32(1) - nodeTemplate.Spec.Replicas = &one - - return nodeTemplate, nil - } - return nil, fmt.Errorf("can't get hollow node template") -} - -func (kubemarkController *KubemarkController) runNodeCreation(stop <-chan struct{}) { - for { - select { - case nodeGroup := <-kubemarkController.createNodeQueue: - kubemarkController.nodeGroupQueueSizeLock.Lock() - err := kubemarkController.addNodeToNodeGroup(nodeGroup) - if err != nil { - klog.Errorf("failed to add node to node group %s: %v", nodeGroup, err) - } else { - kubemarkController.nodeGroupQueueSize[nodeGroup]-- - } - kubemarkController.nodeGroupQueueSizeLock.Unlock() - case <-stop: - return - } - } -} - -func (kubemarkCluster *kubemarkCluster) getHollowNodeName() (string, error) { - selector, _ := labels.Parse(nodeGroupLabel) - nodes, err := kubemarkCluster.nodeLister.List(selector) - if err != nil { - return "", err - } - for _, node := range nodes { - return node.Name, nil - } - return "", fmt.Errorf("did not find any hollow nodes in the cluster") -} - -func (kubemarkCluster *kubemarkCluster) removeUnneededNodes(oldObj interface{}, newObj interface{}) { - node, ok := newObj.(*apiv1.Node) - if !ok { - return - } - for _, condition := range node.Status.Conditions { - // Delete node if it is in unready state, and it has been - // explicitly marked for deletion. - if condition.Type == apiv1.NodeReady && condition.Status != apiv1.ConditionTrue { - kubemarkCluster.nodesToDeleteLock.Lock() - defer kubemarkCluster.nodesToDeleteLock.Unlock() - if kubemarkCluster.nodesToDelete[node.Name] { - kubemarkCluster.nodesToDelete[node.Name] = false - if err := kubemarkCluster.client.CoreV1().Nodes().Delete(context.TODO(), node.Name, metav1.DeleteOptions{}); err != nil { - klog.Errorf("failed to delete node %s from kubemark cluster, err: %v", node.Name, err) - } - } - return - } - } -} - -func (kubemarkCluster *kubemarkCluster) markNodeForDeletion(name string) { - kubemarkCluster.nodesToDeleteLock.Lock() - defer kubemarkCluster.nodesToDeleteLock.Unlock() - kubemarkCluster.nodesToDelete[name] = true -} - -func newReplicationControllerInformer(kubeClient kubeclient.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - rcListWatch := cache.NewListWatchFromClient(kubeClient.CoreV1().RESTClient(), "replicationcontrollers", namespaceKubemark, fields.Everything()) - return cache.NewSharedIndexInformer(rcListWatch, &apiv1.ReplicationController{}, resyncPeriod, nil) -} - -func newPodInformer(kubeClient kubeclient.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - podListWatch := cache.NewListWatchFromClient(kubeClient.CoreV1().RESTClient(), "pods", namespaceKubemark, fields.Everything()) - return cache.NewSharedIndexInformer(podListWatch, &apiv1.Pod{}, resyncPeriod, nil) -} diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go deleted file mode 100644 index bab6044c53e..00000000000 --- a/pkg/kubemark/hollow_kubelet.go +++ /dev/null @@ -1,213 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubemark - -import ( - "context" - "fmt" - "time" - - noopoteltrace "go.opentelemetry.io/otel/trace/noop" - v1 "k8s.io/api/core/v1" - "k8s.io/klog/v2" - "k8s.io/mount-utils" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/record" - internalapi "k8s.io/cri-api/pkg/apis" - kubeletapp "k8s.io/kubernetes/cmd/kubelet/app" - "k8s.io/kubernetes/cmd/kubelet/app/options" - "k8s.io/kubernetes/pkg/kubelet" - kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" - "k8s.io/kubernetes/pkg/kubelet/cadvisor" - "k8s.io/kubernetes/pkg/kubelet/cm" - containertest "k8s.io/kubernetes/pkg/kubelet/container/testing" - probetest "k8s.io/kubernetes/pkg/kubelet/prober/testing" - kubetypes "k8s.io/kubernetes/pkg/kubelet/types" - kubeletutil "k8s.io/kubernetes/pkg/kubelet/util" - "k8s.io/kubernetes/pkg/util/oom" - "k8s.io/kubernetes/pkg/volume" - "k8s.io/kubernetes/pkg/volume/configmap" - "k8s.io/kubernetes/pkg/volume/csi" - "k8s.io/kubernetes/pkg/volume/downwardapi" - "k8s.io/kubernetes/pkg/volume/emptydir" - "k8s.io/kubernetes/pkg/volume/fc" - "k8s.io/kubernetes/pkg/volume/git_repo" - "k8s.io/kubernetes/pkg/volume/hostpath" - "k8s.io/kubernetes/pkg/volume/iscsi" - "k8s.io/kubernetes/pkg/volume/local" - "k8s.io/kubernetes/pkg/volume/nfs" - "k8s.io/kubernetes/pkg/volume/projected" - "k8s.io/kubernetes/pkg/volume/secret" - "k8s.io/kubernetes/pkg/volume/util/hostutil" - "k8s.io/kubernetes/pkg/volume/util/subpath" - "k8s.io/kubernetes/test/utils" -) - -type HollowKubelet struct { - KubeletFlags *options.KubeletFlags - KubeletConfiguration *kubeletconfig.KubeletConfiguration - KubeletDeps *kubelet.Dependencies -} - -func volumePlugins() []volume.VolumePlugin { - allPlugins := []volume.VolumePlugin{} - allPlugins = append(allPlugins, emptydir.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, hostpath.FakeProbeVolumePlugins(volume.VolumeConfig{})...) - allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(volume.VolumeConfig{})...) - allPlugins = append(allPlugins, secret.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, downwardapi.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, projected.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, local.ProbeVolumePlugins()...) - allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...) - return allPlugins -} - -func NewHollowKubelet( - flags *options.KubeletFlags, - config *kubeletconfig.KubeletConfiguration, - client *clientset.Clientset, - heartbeatClient *clientset.Clientset, - cadvisorInterface cadvisor.Interface, - imageService internalapi.ImageManagerService, - runtimeService internalapi.RuntimeService, - containerManager cm.ContainerManager) *HollowKubelet { - // Kubemark dependency construction does not have a higher-level logger. - nodeStartupLatencyTracker := kubeletutil.NewNodeStartupLatencyTracker(klog.TODO()) - - d := &kubelet.Dependencies{ - KubeClient: client, - HeartbeatClient: heartbeatClient, - ProbeManager: probetest.FakeManager{}, - RemoteRuntimeService: runtimeService, - RemoteImageService: imageService, - CAdvisorInterface: cadvisorInterface, - OSInterface: &containertest.FakeOS{}, - ContainerManager: containerManager, - VolumePlugins: volumePlugins(), - TLSOptions: nil, - OOMAdjuster: oom.NewFakeOOMAdjuster(), - Mounter: &mount.FakeMounter{}, - Subpather: &subpath.FakeSubpath{}, - HostUtil: hostutil.NewFakeHostUtil(nil), - PodStartupLatencyTracker: kubeletutil.NewPodStartupLatencyTracker(), - NodeStartupLatencyTracker: nodeStartupLatencyTracker, - TracerProvider: noopoteltrace.NewTracerProvider(), - Recorder: &record.FakeRecorder{}, // With real recorder we attempt to read /dev/kmsg. - } - - return &HollowKubelet{ - KubeletFlags: flags, - KubeletConfiguration: config, - KubeletDeps: d, - } -} - -// Starts this HollowKubelet and blocks. -func (hk *HollowKubelet) Run(ctx context.Context) { - if err := kubeletapp.RunKubelet(ctx, &options.KubeletServer{ - KubeletFlags: *hk.KubeletFlags, - KubeletConfiguration: *hk.KubeletConfiguration, - }, hk.KubeletDeps); err != nil { - klog.Fatalf("Failed to run HollowKubelet: %v. Exiting.", err) - } - select {} -} - -// HollowKubeletOptions contains settable parameters for hollow kubelet. -type HollowKubeletOptions struct { - NodeName string - KubeletPort int - KubeletReadOnlyPort int - MaxPods int - PodsPerCore int - NodeLabels map[string]string - RegisterWithTaints []v1.Taint -} - -// Builds a KubeletConfiguration for the HollowKubelet, ensuring that the -// usual defaults are applied for fields we do not override. -func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, *kubeletconfig.KubeletConfiguration) { - testRootDir := utils.MakeTempDirOrDie("hollow-kubelet.", "") - podFilePath := utils.MakeTempDirOrDie("static-pods", testRootDir) - podLogsPath := utils.MakeTempDirOrDie("pod-logs", testRootDir) - klog.Infof("Using %s as root dir for hollow-kubelet", testRootDir) - - // Flags struct - f := options.NewKubeletFlags() - f.RootDirectory = testRootDir - f.HostnameOverride = opt.NodeName - f.MinimumGCAge = metav1.Duration{Duration: 1 * time.Minute} - f.MaxContainerCount = 100 - f.MaxPerPodContainerCount = 2 - f.NodeLabels = opt.NodeLabels - - // Config struct - c, err := options.NewKubeletConfiguration() - if err != nil { - panic(err) - } - - c.ImageServiceEndpoint = "unix:///run/containerd/containerd.sock" - c.StaticPodURL = "" - c.EnableServer = true - c.Address = "0.0.0.0" /* bind address */ - c.Port = int32(opt.KubeletPort) - c.ReadOnlyPort = int32(opt.KubeletReadOnlyPort) - c.StaticPodPath = podFilePath - c.FileCheckFrequency.Duration = 20 * time.Second - c.HTTPCheckFrequency.Duration = 20 * time.Second - c.NodeStatusUpdateFrequency.Duration = 10 * time.Second - c.NodeStatusReportFrequency.Duration = 5 * time.Minute - c.SyncFrequency.Duration = 10 * time.Second - c.EvictionPressureTransitionPeriod.Duration = 5 * time.Minute - c.MaxPods = int32(opt.MaxPods) - c.PodsPerCore = int32(opt.PodsPerCore) - c.ClusterDNS = []string{} - c.ImageGCHighThresholdPercent = 90 - c.ImageGCLowThresholdPercent = 80 - c.ProviderID = fmt.Sprintf("kubemark://%v", opt.NodeName) - c.VolumeStatsAggPeriod.Duration = time.Minute - c.CgroupRoot = "" - c.CPUCFSQuota = true - c.EnableControllerAttachDetach = false - c.EnableDebuggingHandlers = true - c.CgroupsPerQOS = false - // hairpin-veth is used to allow hairpin packets. Note that this deviates from - // what the "real" kubelet currently does, because there's no way to - // set promiscuous mode on docker0. - c.HairpinMode = kubeletconfig.HairpinVeth - c.MaxOpenFiles = 1024 - c.RegistryBurst = 10 - c.RegistryPullQPS = 5.0 - c.ResolverConfig = kubetypes.ResolvConfDefault - c.KubeletCgroups = "/kubelet" - c.SerializeImagePulls = true - c.SystemCgroups = "" - c.ProtectKernelDefaults = false - c.RegisterWithTaints = opt.RegisterWithTaints - c.RegisterNode = true - c.LocalStorageCapacityIsolation = true - c.PodLogsDir = podLogsPath - - return f, c -} diff --git a/pkg/proxy/kubemark/.import-restrictions b/pkg/proxy/kubemark/.import-restrictions deleted file mode 100644 index d333ec6b4fa..00000000000 --- a/pkg/proxy/kubemark/.import-restrictions +++ /dev/null @@ -1,6 +0,0 @@ -rules: - # override pkg/ import restriction on cmd/ for kubemark - - selectorRegexp: k8s[.]io/kubernetes/cmd - allowedPrefixes: - - k8s.io/kubernetes/cmd/kube-proxy/app - diff --git a/pkg/proxy/kubemark/OWNERS b/pkg/proxy/kubemark/OWNERS deleted file mode 100644 index 16eb9516d79..00000000000 --- a/pkg/proxy/kubemark/OWNERS +++ /dev/null @@ -1,14 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: - - shyamjvs - - sig-scalability-reviewers - - wojtek-t -approvers: - - shyamjvs - - sig-scalability-approvers - - wojtek-t -emeritus_approvers: - - gmarek -labels: - - sig/scalability diff --git a/pkg/proxy/kubemark/hollow_proxy.go b/pkg/proxy/kubemark/hollow_proxy.go deleted file mode 100644 index 7b7901dfe8a..00000000000 --- a/pkg/proxy/kubemark/hollow_proxy.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubemark - -import ( - "context" - "fmt" - "time" - - v1 "k8s.io/api/core/v1" - discoveryv1 "k8s.io/api/discovery/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientset "k8s.io/client-go/kubernetes" - v1core "k8s.io/client-go/kubernetes/typed/core/v1" - "k8s.io/client-go/tools/events" - proxyapp "k8s.io/kubernetes/cmd/kube-proxy/app" - kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config" - "k8s.io/utils/ptr" -) - -type HollowProxy struct { - ProxyServer *proxyapp.ProxyServer -} - -type FakeProxier struct{} - -func (*FakeProxier) Sync() {} -func (*FakeProxier) SyncLoop() { - select {} -} -func (*FakeProxier) OnServiceAdd(service *v1.Service) {} -func (*FakeProxier) OnServiceUpdate(oldService, service *v1.Service) {} -func (*FakeProxier) OnServiceDelete(service *v1.Service) {} -func (*FakeProxier) OnServiceSynced() {} -func (*FakeProxier) OnEndpointSliceAdd(slice *discoveryv1.EndpointSlice) {} -func (*FakeProxier) OnEndpointSliceUpdate(oldSlice, slice *discoveryv1.EndpointSlice) {} -func (*FakeProxier) OnEndpointSliceDelete(slice *discoveryv1.EndpointSlice) {} -func (*FakeProxier) OnEndpointSlicesSynced() {} -func (*FakeProxier) OnServiceCIDRsChanged(_ []string) {} -func (*FakeProxier) OnTopologyChange(_ map[string]string) {} - -func NewHollowProxy( - nodeName string, - client clientset.Interface, - eventClient v1core.EventsGetter, - broadcaster events.EventBroadcaster, - recorder events.EventRecorder, -) *HollowProxy { - return &HollowProxy{ - ProxyServer: &proxyapp.ProxyServer{ - Config: &kubeproxyconfig.KubeProxyConfiguration{ - Mode: kubeproxyconfig.ProxyMode("fake"), - ConfigSyncPeriod: metav1.Duration{Duration: 30 * time.Second}, - Linux: kubeproxyconfig.KubeProxyLinuxConfiguration{ - OOMScoreAdj: ptr.To[int32](0), - }, - }, - - Client: client, - Proxier: &FakeProxier{}, - Broadcaster: broadcaster, - Recorder: recorder, - NodeRef: &v1.ObjectReference{ - APIVersion: "v1", - Kind: "Node", - Name: nodeName, - Namespace: "", - }, - }, - } -} - -func (hp *HollowProxy) Run() error { - - if err := hp.ProxyServer.Run(context.TODO()); err != nil { - return fmt.Errorf("Error while running proxy: %w", err) - } - return nil -} diff --git a/test/e2e/apimachinery/garbage_collector.go b/test/e2e/apimachinery/garbage_collector.go index 52626551817..4743bd25d8d 100644 --- a/test/e2e/apimachinery/garbage_collector.go +++ b/test/e2e/apimachinery/garbage_collector.go @@ -261,7 +261,7 @@ func verifyRemainingObjects(ctx context.Context, f *framework.Framework, objects func gatherMetrics(ctx context.Context, f *framework.Framework) { ginkgo.By("Gathering metrics") var summary framework.TestDataSummary - grabber, err := e2emetrics.NewMetricsGrabber(ctx, f.ClientSet, f.KubemarkExternalClusterClientSet, f.ClientConfig(), false, false, true, false, false, false) + grabber, err := e2emetrics.NewMetricsGrabber(ctx, f.ClientSet, f.ClientConfig(), false, false, true, false, false, false) if err != nil { framework.Logf("Failed to create MetricsGrabber. Skipping metrics gathering.") } else { diff --git a/test/e2e/framework/debug/init/init.go b/test/e2e/framework/debug/init/init.go index adfcb3242a3..91e52503b79 100644 --- a/test/e2e/framework/debug/init/init.go +++ b/test/e2e/framework/debug/init/init.go @@ -69,7 +69,6 @@ func init() { } gatherer, err := e2edebug.NewResourceUsageGatherer(ctx, f.ClientSet, e2edebug.ResourceGathererOptions{ - InKubemark: framework.ProviderIs("kubemark"), Nodes: nodeMode, ResourceDataGatheringPeriod: 60 * time.Second, ProbeDuration: 15 * time.Second, diff --git a/test/e2e/framework/debug/resource_usage_gatherer.go b/test/e2e/framework/debug/resource_usage_gatherer.go index aed88d5188f..b2a767c7b42 100644 --- a/test/e2e/framework/debug/resource_usage_gatherer.go +++ b/test/e2e/framework/debug/resource_usage_gatherer.go @@ -17,7 +17,6 @@ limitations under the License. package debug import ( - "bufio" "bytes" "context" "encoding/json" @@ -40,7 +39,6 @@ import ( kubeletstatsv1alpha1 "k8s.io/kubelet/pkg/apis/stats/v1alpha1" "k8s.io/kubernetes/test/e2e/framework" - e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" "k8s.io/utils/ptr" ) @@ -177,7 +175,6 @@ type resourceGatherWorker struct { stopCh chan struct{} dataSeries []ResourceUsagePerContainer finished bool - inKubemark bool resourceDataGatheringPeriod time.Duration probeDuration time.Duration printVerboseLogs bool @@ -185,29 +182,15 @@ type resourceGatherWorker struct { func (w *resourceGatherWorker) singleProbe(ctx context.Context) { data := make(ResourceUsagePerContainer) - if w.inKubemark { - kubemarkData := getKubemarkMasterComponentsResourceUsage(ctx) - if kubemarkData == nil { - return - } - for k, v := range kubemarkData { - data[k] = &ContainerResourceUsage{ - Name: v.Name, - MemoryWorkingSetInBytes: v.MemoryWorkingSetInBytes, - CPUUsageInCores: v.CPUUsageInCores, - } - } - } else { - nodeUsage, err := getOneTimeResourceUsageOnNode(w.c, w.nodeName, w.probeDuration, func() []string { return w.containerIDs }) - if err != nil { - framework.Logf("Error while reading data from %v: %v", w.nodeName, err) - return - } - for k, v := range nodeUsage { - data[k] = v - if w.printVerboseLogs { - framework.Logf("Get container %v usage on node %v. CPUUsageInCores: %v, MemoryUsageInBytes: %v, MemoryWorkingSetInBytes: %v", k, w.nodeName, v.CPUUsageInCores, v.MemoryUsageInBytes, v.MemoryWorkingSetInBytes) - } + nodeUsage, err := getOneTimeResourceUsageOnNode(w.c, w.nodeName, w.probeDuration, func() []string { return w.containerIDs }) + if err != nil { + framework.Logf("Error while reading data from %v: %v", w.nodeName, err) + return + } + for k, v := range nodeUsage { + data[k] = v + if w.printVerboseLogs { + framework.Logf("Get container %v usage on node %v. CPUUsageInCores: %v, MemoryUsageInBytes: %v, MemoryWorkingSetInBytes: %v", k, w.nodeName, v.CPUUsageInCores, v.MemoryUsageInBytes, v.MemoryWorkingSetInBytes) } } w.dataSeries = append(w.dataSeries, data) @@ -351,7 +334,6 @@ type ContainerResourceGatherer struct { // ResourceGathererOptions is a struct to hold options for resource. type ResourceGathererOptions struct { - InKubemark bool Nodes NodesSet ResourceDataGatheringPeriod time.Duration ProbeDuration time.Duration @@ -402,20 +384,6 @@ func NewResourceUsageGatherer(ctx context.Context, c clientset.Interface, option options: options, } - if options.InKubemark { - g.workerWg.Add(1) - g.workers = append(g.workers, resourceGatherWorker{ - inKubemark: true, - stopCh: g.stopCh, - wg: &g.workerWg, - finished: false, - resourceDataGatheringPeriod: options.ResourceDataGatheringPeriod, - probeDuration: options.ProbeDuration, - printVerboseLogs: options.PrintVerboseLogs, - }) - return &g, nil - } - // Tracks kube-system pods if no valid PodList is passed in. var err error if pods == nil { @@ -475,7 +443,6 @@ func NewResourceUsageGatherer(ctx context.Context, c clientset.Interface, option containerIDs: g.containerIDs, stopCh: g.stopCh, finished: false, - inKubemark: false, resourceDataGatheringPeriod: options.ResourceDataGatheringPeriod, probeDuration: options.ProbeDuration, printVerboseLogs: options.PrintVerboseLogs, @@ -594,65 +561,3 @@ func (g *ContainerResourceGatherer) StopAndSummarize(percentiles []int, constrai } return &summary, nil } - -// kubemarkResourceUsage is a struct for tracking the resource usage of kubemark. -type kubemarkResourceUsage struct { - Name string - MemoryWorkingSetInBytes uint64 - CPUUsageInCores float64 -} - -func getMasterUsageByPrefix(ctx context.Context, prefix string) (string, error) { - sshResult, err := e2essh.SSH(ctx, fmt.Sprintf("ps ax -o %%cpu,rss,command | tail -n +2 | grep %v | sed 's/\\s+/ /g'", prefix), framework.APIAddress()+":22", framework.TestContext.Provider) - if err != nil { - return "", err - } - return sshResult.Stdout, nil -} - -// getKubemarkMasterComponentsResourceUsage returns the resource usage of kubemark which contains multiple combinations of cpu and memory usage for each pod name. -func getKubemarkMasterComponentsResourceUsage(ctx context.Context) map[string]*kubemarkResourceUsage { - result := make(map[string]*kubemarkResourceUsage) - // Get kubernetes component resource usage - sshResult, err := getMasterUsageByPrefix(ctx, "kube") - if err != nil { - framework.Logf("Error when trying to SSH to master machine. Skipping probe. %v", err) - return nil - } - scanner := bufio.NewScanner(strings.NewReader(sshResult)) - for scanner.Scan() { - var cpu float64 - var mem uint64 - var name string - fmt.Sscanf(strings.TrimSpace(scanner.Text()), "%f %d /usr/local/bin/kube-%s", &cpu, &mem, &name) - if name != "" { - // Gatherer expects pod_name/container_name format - fullName := name + "/" + name - result[fullName] = &kubemarkResourceUsage{Name: fullName, MemoryWorkingSetInBytes: mem * 1024, CPUUsageInCores: cpu / 100} - } - } - // Get etcd resource usage - sshResult, err = getMasterUsageByPrefix(ctx, "bin/etcd") - if err != nil { - framework.Logf("Error when trying to SSH to master machine. Skipping probe") - return nil - } - scanner = bufio.NewScanner(strings.NewReader(sshResult)) - for scanner.Scan() { - var cpu float64 - var mem uint64 - var etcdKind string - fmt.Sscanf(strings.TrimSpace(scanner.Text()), "%f %d /bin/sh -c /usr/local/bin/etcd", &cpu, &mem) - dataDirStart := strings.Index(scanner.Text(), "--data-dir") - if dataDirStart < 0 { - continue - } - fmt.Sscanf(scanner.Text()[dataDirStart:], "--data-dir=/var/%s", &etcdKind) - if etcdKind != "" { - // Gatherer expects pod_name/container_name format - fullName := "etcd/" + etcdKind - result[fullName] = &kubemarkResourceUsage{Name: fullName, MemoryWorkingSetInBytes: mem * 1024, CPUUsageInCores: cpu / 100} - } - } - return result -} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 1d2649df092..dcdba00b022 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -114,10 +114,9 @@ type Framework struct { // test multiple times in parallel. UniqueName string - clientConfig *rest.Config - restMapper *restmapper.DeferredDiscoveryRESTMapper - ClientSet clientset.Interface - KubemarkExternalClusterClientSet clientset.Interface + clientConfig *rest.Config + restMapper *restmapper.DeferredDiscoveryRESTMapper + ClientSet clientset.Interface DynamicClient dynamic.Interface diff --git a/test/e2e/framework/metrics/grab.go b/test/e2e/framework/metrics/grab.go index 23e9ad2b4b9..5f65c0aa2d6 100644 --- a/test/e2e/framework/metrics/grab.go +++ b/test/e2e/framework/metrics/grab.go @@ -31,7 +31,7 @@ func GrabBeforeEach(ctx context.Context, f *framework.Framework) (result *Collec } ginkgo.By("Gathering metrics before test", func() { - grabber, err := NewMetricsGrabber(ctx, f.ClientSet, f.KubemarkExternalClusterClientSet, f.ClientConfig(), !framework.ProviderIs("kubemark"), false, false, false, framework.TestContext.IncludeClusterAutoscalerMetrics, false) + grabber, err := NewMetricsGrabber(ctx, f.ClientSet, f.ClientConfig(), true, false, false, false, framework.TestContext.IncludeClusterAutoscalerMetrics, false) if err != nil { framework.Logf("Failed to create MetricsGrabber (skipping ClusterAutoscaler metrics gathering before test): %v", err) return @@ -55,8 +55,8 @@ func GrabAfterEach(ctx context.Context, f *framework.Framework, before *Collecti ginkgo.By("Gathering metrics after test", func() { // Grab apiserver, scheduler, controller-manager metrics and (optionally) nodes' kubelet metrics. - grabMetricsFromKubelets := framework.TestContext.GatherMetricsAfterTest != "master" && !framework.ProviderIs("kubemark") - grabber, err := NewMetricsGrabber(ctx, f.ClientSet, f.KubemarkExternalClusterClientSet, f.ClientConfig(), grabMetricsFromKubelets, true, true, true, framework.TestContext.IncludeClusterAutoscalerMetrics, false) + grabMetricsFromKubelets := framework.TestContext.GatherMetricsAfterTest != "master" + grabber, err := NewMetricsGrabber(ctx, f.ClientSet, f.ClientConfig(), grabMetricsFromKubelets, true, true, true, framework.TestContext.IncludeClusterAutoscalerMetrics, false) if err != nil { framework.Logf("Failed to create MetricsGrabber (skipping metrics gathering): %v", err) return diff --git a/test/e2e/framework/metrics/kubelet_metrics.go b/test/e2e/framework/metrics/kubelet_metrics.go index ae7a7e211e0..9bedc68b9e7 100644 --- a/test/e2e/framework/metrics/kubelet_metrics.go +++ b/test/e2e/framework/metrics/kubelet_metrics.go @@ -118,7 +118,7 @@ func getKubeletMetricsFromNode(ctx context.Context, c clientset.Interface, nodeN if c == nil { return GrabKubeletMetricsWithoutProxy(ctx, nodeName, "/metrics") } - grabber, err := NewMetricsGrabber(ctx, c, nil, nil, true, false, false, false, false, false) + grabber, err := NewMetricsGrabber(ctx, c, nil, true, false, false, false, false, false) if err != nil { return KubeletMetrics{}, err } diff --git a/test/e2e/framework/metrics/metrics_grabber.go b/test/e2e/framework/metrics/metrics_grabber.go index b49889aa236..1adcac178a5 100644 --- a/test/e2e/framework/metrics/metrics_grabber.go +++ b/test/e2e/framework/metrics/metrics_grabber.go @@ -70,7 +70,6 @@ type Collection struct { // Grabber provides functions which grab metrics from components type Grabber struct { client clientset.Interface - externalClient clientset.Interface config *rest.Config grabFromAPIServer bool grabFromControllerManager bool @@ -94,7 +93,7 @@ type Grabber struct { // Collecting metrics data is an optional debug feature. Not all clusters will // support it. If disabled for a component, the corresponding Grab function // will immediately return an error derived from MetricsGrabbingDisabledError. -func NewMetricsGrabber(ctx context.Context, c clientset.Interface, ec clientset.Interface, config *rest.Config, kubelets bool, scheduler bool, controllers bool, apiServer bool, clusterAutoscaler bool, snapshotController bool) (*Grabber, error) { +func NewMetricsGrabber(ctx context.Context, c clientset.Interface, config *rest.Config, kubelets bool, scheduler bool, controllers bool, apiServer bool, clusterAutoscaler bool, snapshotController bool) (*Grabber, error) { kubeScheduler := "" kubeControllerManager := "" snapshotControllerManager := "" @@ -128,13 +127,9 @@ func NewMetricsGrabber(ctx context.Context, c clientset.Interface, ec clientset. break } } - if clusterAutoscaler && ec == nil { - klog.Warningf("Did not receive an external client interface. Grabbing metrics from ClusterAutoscaler is disabled.") - } return &Grabber{ client: c, - externalClient: ec, config: config, grabFromAPIServer: apiServer, grabFromControllerManager: checkPodDebugHandlers(ctx, c, controllers, "kube-controller-manager", kubeControllerManager), @@ -309,19 +304,10 @@ func (g *Grabber) GrabFromScheduler(ctx context.Context) (SchedulerMetrics, erro // GrabFromClusterAutoscaler returns metrics from cluster autoscaler func (g *Grabber) GrabFromClusterAutoscaler(ctx context.Context) (ClusterAutoscalerMetrics, error) { - if !g.HasControlPlanePods() && g.externalClient == nil { + if !g.HasControlPlanePods() { return ClusterAutoscalerMetrics{}, fmt.Errorf("ClusterAutoscaler: %w", MetricsGrabbingDisabledError) } - var client clientset.Interface - var namespace string - if g.externalClient != nil { - client = g.externalClient - namespace = "kubemark" - } else { - client = g.client - namespace = metav1.NamespaceSystem - } - output, err := g.getMetricsFromPod(ctx, client, "cluster-autoscaler", namespace, 8085) + output, err := g.getMetricsFromPod(ctx, g.client, "cluster-autoscaler", metav1.NamespaceSystem, 8085) if err != nil { return ClusterAutoscalerMetrics{}, err } diff --git a/test/e2e/framework/providers/kubemark/.import-restrictions b/test/e2e/framework/providers/kubemark/.import-restrictions deleted file mode 100644 index fdd90e600b5..00000000000 --- a/test/e2e/framework/providers/kubemark/.import-restrictions +++ /dev/null @@ -1,9 +0,0 @@ -# This E2E framework sub-package is currently allowed to use arbitrary -# dependencies, therefore we need to override the restrictions from -# the parent .import-restrictions file. -# -# At some point it may become useful to also check this package's -# dependencies more careful. -rules: - - selectorRegexp: "" - allowedPrefixes: [ "" ] diff --git a/test/e2e/framework/providers/kubemark/kubemark.go b/test/e2e/framework/providers/kubemark/kubemark.go deleted file mode 100644 index 4ac202a8999..00000000000 --- a/test/e2e/framework/providers/kubemark/kubemark.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubemark - -import ( - "flag" - "fmt" - - "k8s.io/client-go/informers" - clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "k8s.io/kubernetes/pkg/kubemark" - "k8s.io/kubernetes/test/e2e/framework" -) - -var ( - kubemarkExternalKubeConfig = flag.String(fmt.Sprintf("%s-%s", "kubemark-external", clientcmd.RecommendedConfigPathFlag), "", "Path to kubeconfig containing embedded authinfo for external cluster.") -) - -func init() { - framework.RegisterProvider("kubemark", newProvider) -} - -func newProvider() (framework.ProviderInterface, error) { - // Actual initialization happens when the e2e framework gets constructed. - return &Provider{}, nil -} - -// Provider is a structure to handle Kubemark cluster for e2e testing -type Provider struct { - framework.NullProvider - controller *kubemark.KubemarkController - closeChannel chan struct{} -} - -// ResizeGroup resizes an instance group -func (p *Provider) ResizeGroup(group string, size int32) error { - return p.controller.SetNodeGroupSize(group, int(size)) -} - -// GetGroupNodes returns a node name for the specified node group -func (p *Provider) GetGroupNodes(group string) ([]string, error) { - return p.controller.GetNodeNamesForNodeGroup(group) -} - -// FrameworkBeforeEach prepares clients, configurations etc. for e2e testing -func (p *Provider) FrameworkBeforeEach(f *framework.Framework) { - if *kubemarkExternalKubeConfig != "" && p.controller == nil { - externalConfig, err := clientcmd.BuildConfigFromFlags("", *kubemarkExternalKubeConfig) - externalConfig.QPS = f.Options.ClientQPS - externalConfig.Burst = f.Options.ClientBurst - framework.ExpectNoError(err) - externalClient, err := clientset.NewForConfig(externalConfig) - framework.ExpectNoError(err) - f.KubemarkExternalClusterClientSet = externalClient - p.closeChannel = make(chan struct{}) - externalInformerFactory := informers.NewSharedInformerFactory(externalClient, 0) - kubemarkInformerFactory := informers.NewSharedInformerFactory(f.ClientSet, 0) - kubemarkNodeInformer := kubemarkInformerFactory.Core().V1().Nodes() - go kubemarkNodeInformer.Informer().Run(p.closeChannel) - p.controller, err = kubemark.NewKubemarkController(externalClient, externalInformerFactory, f.ClientSet, kubemarkNodeInformer) - framework.ExpectNoError(err) - externalInformerFactory.Start(p.closeChannel) - if !p.controller.WaitForCacheSync(p.closeChannel) { - framework.Failf("Unable to sync caches for %v", p.controller) - } - go p.controller.Run(p.closeChannel) - } -} - -// FrameworkAfterEach cleans up after e2e testing -func (p *Provider) FrameworkAfterEach(f *framework.Framework) { - if p.closeChannel != nil { - close(p.closeChannel) - p.controller = nil - p.closeChannel = nil - } -} - -// GroupSize returns the size of an instance group -func (p *Provider) GroupSize(group string) (int, error) { - return p.controller.GetNodeGroupSize(group) -} diff --git a/test/e2e/framework/ssh/ssh.go b/test/e2e/framework/ssh/ssh.go index c9a136d8d7f..53badc09795 100644 --- a/test/e2e/framework/ssh/ssh.go +++ b/test/e2e/framework/ssh/ssh.go @@ -66,7 +66,7 @@ func GetSigner(provider string) (ssh.Signer, error) { // support. keyfile := "" switch provider { - case "gce", "kubemark": + case "gce": keyfile = os.Getenv("GCE_SSH_KEY") if keyfile == "" { keyfile = os.Getenv("GCE_SSH_PRIVATE_KEY_FILE") diff --git a/test/e2e/instrumentation/metrics.go b/test/e2e/instrumentation/metrics.go index 1ebd46e7fb9..0bae0904921 100644 --- a/test/e2e/instrumentation/metrics.go +++ b/test/e2e/instrumentation/metrics.go @@ -36,14 +36,13 @@ import ( var _ = common.SIGDescribe("Metrics", func() { f := framework.NewDefaultFramework("metrics") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged - var c, ec clientset.Interface + var c clientset.Interface var grabber *e2emetrics.Grabber ginkgo.BeforeEach(func(ctx context.Context) { var err error c = f.ClientSet - ec = f.KubemarkExternalClusterClientSet gomega.Eventually(ctx, func() error { - grabber, err = e2emetrics.NewMetricsGrabber(ctx, c, ec, f.ClientConfig(), true, true, true, true, true, true) + grabber, err = e2emetrics.NewMetricsGrabber(ctx, c, f.ClientConfig(), true, true, true, true, true, true) return err }, 5*time.Minute, 10*time.Second).Should(gomega.BeNil()) }) diff --git a/test/e2e/instrumentation/monitoring/metrics_grabber.go b/test/e2e/instrumentation/monitoring/metrics_grabber.go index 742c0f83d5c..9a0971e690b 100644 --- a/test/e2e/instrumentation/monitoring/metrics_grabber.go +++ b/test/e2e/instrumentation/monitoring/metrics_grabber.go @@ -37,14 +37,13 @@ import ( var _ = instrumentation.SIGDescribe("MetricsGrabber", func() { f := framework.NewDefaultFramework("metrics-grabber") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged - var c, ec clientset.Interface + var c clientset.Interface var grabber *e2emetrics.Grabber ginkgo.BeforeEach(func(ctx context.Context) { var err error c = f.ClientSet - ec = f.KubemarkExternalClusterClientSet gomega.Eventually(ctx, func() error { - grabber, err = e2emetrics.NewMetricsGrabber(ctx, c, ec, f.ClientConfig(), true, true, true, true, true, true) + grabber, err = e2emetrics.NewMetricsGrabber(ctx, c, f.ClientConfig(), true, true, true, true, true, true) if err != nil { return fmt.Errorf("failed to create metrics grabber: %w", err) } diff --git a/test/e2e/network/kube_proxy.go b/test/e2e/network/kube_proxy.go index 58f05e924f1..e8cad6e5940 100644 --- a/test/e2e/network/kube_proxy.go +++ b/test/e2e/network/kube_proxy.go @@ -291,7 +291,7 @@ var _ = common.SIGDescribe("KubeProxy", func() { nodeName := nodes.Items[0].Name metricName := "kubeproxy_iptables_localhost_nodeports_accepted_packets_total" - metricsGrabber, err := e2emetrics.NewMetricsGrabber(ctx, fr.ClientSet, nil, fr.ClientConfig(), false, false, false, false, false, false) + metricsGrabber, err := e2emetrics.NewMetricsGrabber(ctx, fr.ClientSet, fr.ClientConfig(), false, false, false, false, false, false) framework.ExpectNoError(err) // create a pod with host-network for execing diff --git a/test/e2e/providers.go b/test/e2e/providers.go index cf24a5272cb..5c422722882 100644 --- a/test/e2e/providers.go +++ b/test/e2e/providers.go @@ -21,7 +21,6 @@ import ( _ "k8s.io/kubernetes/test/e2e/framework/providers/aws" _ "k8s.io/kubernetes/test/e2e/framework/providers/azure" _ "k8s.io/kubernetes/test/e2e/framework/providers/gce" - _ "k8s.io/kubernetes/test/e2e/framework/providers/kubemark" _ "k8s.io/kubernetes/test/e2e/framework/providers/openstack" _ "k8s.io/kubernetes/test/e2e/framework/providers/vsphere" ) diff --git a/test/e2e/storage/csimock/csi_selinux_mount.go b/test/e2e/storage/csimock/csi_selinux_mount.go index acc2cfd9110..82fa180ea18 100644 --- a/test/e2e/storage/csimock/csi_selinux_mount.go +++ b/test/e2e/storage/csimock/csi_selinux_mount.go @@ -808,7 +808,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics and SELinuxWarningC // Some metrics use CSI driver name as a label, which is "csi-mock-" + the namespace name. volumePluginLabel := "volume_plugin=\"kubernetes.io/csi/csi-mock-" + f.Namespace.Name + "\"" - grabber, err := e2emetrics.NewMetricsGrabber(ctx, f.ClientSet, nil, f.ClientConfig(), true /*kubelet*/, false /*scheduler*/, true /*controllers*/, false /*apiserver*/, false /*autoscaler*/, false /*snapshotController*/) + grabber, err := e2emetrics.NewMetricsGrabber(ctx, f.ClientSet, f.ClientConfig(), true /*kubelet*/, false /*scheduler*/, true /*controllers*/, false /*apiserver*/, false /*autoscaler*/, false /*snapshotController*/) framework.ExpectNoError(err, "creating the metrics grabber") var nodeStageMountOpts, nodePublishMountOpts []string diff --git a/test/e2e/storage/csimock/csi_snapshot.go b/test/e2e/storage/csimock/csi_snapshot.go index b41c6490a95..4fddc67a9aa 100644 --- a/test/e2e/storage/csimock/csi_snapshot.go +++ b/test/e2e/storage/csimock/csi_snapshot.go @@ -310,7 +310,7 @@ var _ = utils.SIGDescribe("CSI Mock volume snapshot", func() { } ginkgo.DeferCleanup(m.cleanup) - metricsGrabber, err := e2emetrics.NewMetricsGrabber(ctx, m.config.Framework.ClientSet, nil, f.ClientConfig(), false, false, false, false, false, true) + metricsGrabber, err := e2emetrics.NewMetricsGrabber(ctx, m.config.Framework.ClientSet, f.ClientConfig(), false, false, false, false, false, true) if err != nil { framework.Failf("Error creating metrics grabber : %v", err) } diff --git a/test/e2e/storage/testsuites/base.go b/test/e2e/storage/testsuites/base.go index ae83b0c2fb6..ae6e40ddc3a 100644 --- a/test/e2e/storage/testsuites/base.go +++ b/test/e2e/storage/testsuites/base.go @@ -125,7 +125,7 @@ func getVolumeOpCounts(ctx context.Context, c clientset.Interface, config *rest. nodeLimit := 25 - metricsGrabber, err := e2emetrics.NewMetricsGrabber(ctx, c, nil, config, true, false, true, false, false, false) + metricsGrabber, err := e2emetrics.NewMetricsGrabber(ctx, c, config, true, false, true, false, false, false) if err != nil { framework.ExpectNoError(err, "Error creating metrics grabber: %v", err) diff --git a/test/e2e/storage/volume_metrics.go b/test/e2e/storage/volume_metrics.go index 51d90bd70c4..2d3943b6385 100644 --- a/test/e2e/storage/volume_metrics.go +++ b/test/e2e/storage/volume_metrics.go @@ -124,7 +124,7 @@ var _ = utils.SIGDescribe("Volume metrics", func() { VolumeMode: ptr.To(v1.PersistentVolumeBlock), }, ns) - metricsGrabber, err = e2emetrics.NewMetricsGrabber(ctx, c, nil, f.ClientConfig(), true, false, true, false, false, false) + metricsGrabber, err = e2emetrics.NewMetricsGrabber(ctx, c, f.ClientConfig(), true, false, true, false, false, false) if err != nil { framework.Failf("Error creating metrics grabber : %v", err) diff --git a/test/e2e/suites.go b/test/e2e/suites.go index ba3e26fd092..bd2fa6b1075 100644 --- a/test/e2e/suites.go +++ b/test/e2e/suites.go @@ -54,8 +54,8 @@ func gatherTestSuiteMetrics(ctx context.Context) error { return fmt.Errorf("error creating client: %w", err) } - // Grab metrics for apiserver, scheduler, controller-manager, kubelet (for non-kubemark case) and cluster autoscaler (optionally). - grabber, err := e2emetrics.NewMetricsGrabber(ctx, c, nil, config, !framework.ProviderIs("kubemark"), true, true, true, framework.TestContext.IncludeClusterAutoscalerMetrics, false) + // Grab metrics for apiserver, scheduler, controller-manager, kubelet and cluster autoscaler (optionally). + grabber, err := e2emetrics.NewMetricsGrabber(ctx, c, config, true, true, true, true, framework.TestContext.IncludeClusterAutoscalerMetrics, false) if err != nil { return fmt.Errorf("failed to create MetricsGrabber: %w", err) } diff --git a/test/kubemark/OWNERS b/test/kubemark/OWNERS deleted file mode 100644 index c9d4da44566..00000000000 --- a/test/kubemark/OWNERS +++ /dev/null @@ -1,13 +0,0 @@ -# See the OWNERS docs at https://go.k8s.io/owners - -reviewers: - - sig-scalability-reviewers - - wojtek-t -approvers: - - sig-scalability-approvers - - wojtek-t -emeritus_approvers: - - gmarek - - shyamjvs -labels: - - sig/scalability diff --git a/test/kubemark/cloud-provider-config.sh b/test/kubemark/cloud-provider-config.sh deleted file mode 100755 index ad7029f2389..00000000000 --- a/test/kubemark/cloud-provider-config.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -CLOUD_PROVIDER="${CLOUD_PROVIDER:-gce}" -CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-gcr.io}" -PROJECT="${PROJECT:-}" -KUBEMARK_IMAGE_REGISTRY="${KUBEMARK_IMAGE_REGISTRY:-}" -KUBEMARK_IMAGE_MAKE_TARGET="${KUBEMARK_IMAGE_MAKE_TARGET:-gcloudpush}" diff --git a/test/kubemark/common/util.sh b/test/kubemark/common/util.sh deleted file mode 100644 index 05cdba4c77c..00000000000 --- a/test/kubemark/common/util.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Running cmd $RETRIES times in case of failures. -function run-cmd-with-retries { - RETRIES="${RETRIES:-3}" - for attempt in $(seq 1 "${RETRIES}"); do - local ret_val=0 - exec 5>&1 # Duplicate &1 to &5 for use below. - # We don't use 'local' to declare result as then ret_val always gets value 0. - # We use tee to output to &5 (redirected to stdout) while also storing it in the variable. - result=$("$@" 2>&1 | tee >(cat - >&5)) || ret_val="$?" - if [[ "${ret_val:-0}" -ne "0" ]]; then - if [[ $(echo "${result}" | grep -c "already exists") -gt 0 ]]; then - if [[ "${attempt}" == 1 ]]; then - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_red}Failed to $1 $2 ${3:-} as the resource hasn't been deleted from a previous run.${color_norm}" >& 2 - exit 1 - fi - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_yellow}Succeeded to $1 $2 ${3:-} in the previous attempt, but status response wasn't received.${color_norm}" - return 0 - fi - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_yellow}Attempt $attempt failed to $1 $2 ${3:-}. Retrying.${color_norm}" >& 2 - sleep $((attempt * 5)) - else - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_green}Succeeded to $1 $2 ${3:-}.${color_norm}" - return 0 - fi - done - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_red}Failed to $1 $2 ${3:-}.${color_norm}" >& 2 - exit 1 -} diff --git a/test/kubemark/configure-kubectl.sh b/test/kubemark/configure-kubectl.sh deleted file mode 100755 index 2ac77b5e73d..00000000000 --- a/test/kubemark/configure-kubectl.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script assumes that kubectl binary is present in PATH. -kubectl config set-cluster hollow-cluster --server=http://localhost:8080 --insecure-skip-tls-verify=true -kubectl config set-credentials "$(whoami)" -kubectl config set-context hollow-context --cluster=hollow-cluster --user="$(whoami)" diff --git a/test/kubemark/gce/util.sh b/test/kubemark/gce/util.sh deleted file mode 100644 index 5e19e95b69a..00000000000 --- a/test/kubemark/gce/util.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../.. - -source "${KUBE_ROOT}/test/kubemark/common/util.sh" - -# Wrapper for gcloud compute, running it $RETRIES times in case of failures. -# Args: -# $@: all stuff that goes after 'gcloud compute' -function run-gcloud-compute-with-retries { - run-cmd-with-retries gcloud compute "$@" -} - -function authenticate-docker { - echo "Configuring registry authentication" - mkdir -p "${HOME}/.docker" - gcloud beta auth configure-docker -q -} - -function create-kubemark-master { - # We intentionally override env vars in subshell to preserve original values. - # shellcheck disable=SC2030,SC2031 - ( - # All calls to e2e-grow-cluster must share temp dir with initial e2e-up.sh. - kube::util::ensure-temp-dir - export KUBE_TEMP="${KUBE_TEMP}" - - export KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark" - export KUBECONFIG_INTERNAL="${RESOURCE_DIRECTORY}/kubeconfig-internal.kubemark" - export CLUSTER_NAME="${CLUSTER_NAME}-kubemark" - export KUBE_CREATE_NODES=false - export KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX}-kubemark" - - # Quite tricky cidr setup: we set KUBE_GCE_ENABLE_IP_ALIASES=true to avoid creating - # cloud routes and RangeAllocator to assign cidrs by kube-controller-manager. - export KUBE_GCE_ENABLE_IP_ALIASES=true - export KUBE_GCE_NODE_IPAM_MODE=RangeAllocator - - # Disable all addons. They are running outside of the kubemark cluster. - export KUBE_ENABLE_CLUSTER_AUTOSCALER=false - export KUBE_ENABLE_CLUSTER_DNS=false - export KUBE_ENABLE_NODE_LOGGING=false - export KUBE_ENABLE_METRICS_SERVER=false - export KUBE_ENABLE_L7_LOADBALANCING="none" - - # Unset env variables set by kubetest for 'root cluster'. We need recompute them - # for kubemark master. - # TODO(mborsz): Figure out some better way to filter out such env variables than - # listing them here. - unset MASTER_SIZE MASTER_DISK_SIZE MASTER_ROOT_DISK_SIZE - - # Set kubemark-specific overrides: - # for each defined env KUBEMARK_X=Y call export X=Y. - for var in ${!KUBEMARK_*}; do - dst_var=${var#KUBEMARK_} - val=${!var} - echo "Setting ${dst_var} to '${val}'" - export "${dst_var}"="${val}" - done - - "${KUBE_ROOT}/hack/e2e-internal/e2e-up.sh" - - if [[ "${KUBEMARK_HA_MASTER:-}" == "true" && -n "${KUBEMARK_MASTER_ADDITIONAL_ZONES:-}" ]]; then - for KUBE_GCE_ZONE in ${KUBEMARK_MASTER_ADDITIONAL_ZONES}; do - KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_REPLICATE_EXISTING_MASTER=true \ - "${KUBE_ROOT}/hack/e2e-internal/e2e-grow-cluster.sh" - done - fi - - # The e2e-up.sh script is not sourced, so we don't have access to variables that - # it sets. Instead, we read data which was written to the KUBE_TEMP directory. - # The cluster-location is either ZONE (say us-east1-a) or REGION (say us-east1). - # To get REGION from location, only first two parts are matched. - REGION=$(grep -o "^[a-z]*-[a-z0-9]*" "${KUBE_TEMP}"/cluster-location.txt) - MASTER_NAME="${KUBE_GCE_INSTANCE_PREFIX}"-master - - if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then - MASTER_INTERNAL_IP=$(gcloud compute addresses describe "${MASTER_NAME}-internal-ip" \ - --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') - fi - MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \ - --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') - - # If cluster uses private master IP, two kubeconfigs are created: - # - kubeconfig with public IP, which will be used to connect to the cluster - # from outside of the cluster network - # - kubeconfig with private IP (called internal kubeconfig), which will be - # used to create hollow nodes. - # - # Note that hollow nodes might use either of these kubeconfigs, but - # using internal one is better from performance and cost perspective, since - # traffic does not need to go through Cloud NAT. - if [[ -n "${MASTER_INTERNAL_IP:-}" ]]; then - echo "Writing internal kubeconfig to '${KUBECONFIG_INTERNAL}'" - ip_regexp=${MASTER_IP//./\\.} # escape ".", so that sed won't treat it as "any char" - sed "s/${ip_regexp}/${MASTER_INTERNAL_IP}/g" "${KUBECONFIG}" > "${KUBECONFIG_INTERNAL}" - fi - ) -} - -function delete-kubemark-master { - # We intentionally override env vars in subshell to preserve original values. - # shellcheck disable=SC2030,SC2031 - ( - export CLUSTER_NAME="${CLUSTER_NAME}-kubemark" - export KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX}-kubemark" - - export KUBE_DELETE_NETWORK=false - - if [[ "${KUBEMARK_HA_MASTER:-}" == "true" && -n "${KUBEMARK_MASTER_ADDITIONAL_ZONES:-}" ]]; then - for KUBE_GCE_ZONE in ${KUBEMARK_MASTER_ADDITIONAL_ZONES}; do - KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_REPLICATE_EXISTING_MASTER=true \ - "${KUBE_ROOT}/hack/e2e-internal/e2e-shrink-cluster.sh" - done - fi - - "${KUBE_ROOT}/hack/e2e-internal/e2e-down.sh" - ) -} - -function calculate-node-labels { - echo "cloud.google.com/metadata-proxy-ready=true" -} diff --git a/test/kubemark/pre-existing/README.md b/test/kubemark/pre-existing/README.md deleted file mode 100644 index 1e6531728dd..00000000000 --- a/test/kubemark/pre-existing/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# Kubemark Pre-existing Provider Guide - -**Kubemark Master** -- A set of Kubernetes control plane components running in a VM - -**Kubernetes Cluster** -- A real Kubernetes Cluster that has master and nodes. The hollow-node pods - are run in this cluster, but appear as nodes to the Kubemark Master - -## Introduction - -Every running Kubemark setup looks like the following: - 1) A running Kubernetes cluster pointed to by the local kubeconfig - 2) A separate VM where the kubemark master is running - 3) Some hollow-nodes that run on the Kubernetes Cluster from #1 - 4) The hollow-nodes are configured to talk with the kubemark master at #2 - -When using the pre-existing provider, the developer is responsible for creating -#1 and #2. Therefore, the kubemark scripts will not create any infrastructure -or start a kubemark master like in other providers. Instead, the existing -resources provided by the VM at $MASTER_IP will serve as the kubemark master. - -## Use Case - -The goal of the pre-existing provider is to use the kubemark tools with an -existing kubemark master. It's meant to provide the developer with -additional flexibility to customize the cluster infrastructure and still use -the kubemark setup tools. The pre-existing provider is an **advanced** use -case that requires the developer to have knowledge of setting up a kubemark -master. - -## Requirements - -To use the pre-existing provider, the expectation is that there's a kubemark -master that is reachable at $MASTER_IP. The machine that the kubemark master is -on has to be ssh able from the host that's executing the kubemark scripts. And -the user on that machine has to be 'kubernetes'. - -Requirement checklist: -- Set MASTER_IP to ip address to the kubemark master -- The host where you execute the kubemark scripts must be able to ssh to - kubernetes@$MASTER_IP - -## Example Configuration - -_test/kubemark/cloud-provider-config.sh_ - -``` -CLOUD_PROVIDER="pre-existing" -KUBEMARK_IMAGE_MAKE_TARGET="push" -CONTAINER_REGISTRY=docker.io -PROJECT="rthallisey" -MASTER_IP="192.168.121.29:6443" -``` diff --git a/test/kubemark/pre-existing/util.sh b/test/kubemark/pre-existing/util.sh deleted file mode 100644 index 3e279af5e92..00000000000 --- a/test/kubemark/pre-existing/util.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../.. - -source "${KUBE_ROOT}/test/kubemark/common/util.sh" - -# Leave the skeleton definition of execute-cmd-on-master-with-retries -# so only the pre-existing provider functions will target this. -function execute-cmd-on-pre-existing-master-with-retries() { - IP_WITHOUT_PORT=$(echo "${MASTER_IP}" | cut -f 1 -d ':') || "${MASTER_IP}" - - RETRIES="${2:-1}" run-cmd-with-retries ssh kubernetes@"${IP_WITHOUT_PORT}" "${1}" -} diff --git a/test/kubemark/resources/cluster-autoscaler_template.json b/test/kubemark/resources/cluster-autoscaler_template.json deleted file mode 100644 index 87e465a5f61..00000000000 --- a/test/kubemark/resources/cluster-autoscaler_template.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "kind": "Pod", - "apiVersion": "v1", - "metadata": { - "name": "cluster-autoscaler", - "namespace": "kubemark", - "labels": { - "tier": "cluster-management", - "component": "cluster-autoscaler" - } - }, - "spec": { - "hostNetwork": true, - "containers": [ - { - "name": "cluster-autoscaler", - "image": "registry.k8s.io/cluster-autoscaler:v1.0.0", - "command": [ - "./run.sh", - "--kubernetes=https://{{master_ip}}:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/cluster_autoscaler.kubeconfig", - "--v=4", - "--logtostderr=true", - "--write-status-configmap=true", - "--cloud-provider=kubemark", - "--nodes={{kubemark_autoscaler_min_nodes}}:{{kubemark_autoscaler_max_nodes}}:{{kubemark_autoscaler_mig_name}}" - ], - "env": [ - { - "name": "LOG_OUTPUT", - "value": "/var/log/cluster-autoscaler.log" - } - ], - "resources": { - "requests": { - "cpu": "10m", - "memory": "300Mi" - } - }, - "volumeMounts": [ - {"name": "cloudconfigmount","mountPath": "/etc/gce.conf", "readOnly": true}, - { - "name": "ssl-certs", - "readOnly": true, - "mountPath": "/etc/ssl/certs" - }, - { - "name": "usrsharecacerts", - "readOnly": true, - "mountPath": "/usr/share/ca-certificates" - }, - { - "name": "logdir", - "mountPath": "/var/log", - "readOnly": false - }, - { - "name": "kubeconfig-volume", - "mountPath": "/kubeconfig" - } - ], - "terminationMessagePath": "/dev/termination-log", - "imagePullPolicy": "Always" - } - ], - "volumes": [ - {"name": "cloudconfigmount","hostPath": {"path": "/etc/gce.conf"}}, - { - "name": "ssl-certs", - "hostPath": { - "path": "/etc/ssl/certs" - } - }, - { - "name": "usrsharecacerts", - "hostPath": { - "path": "/usr/share/ca-certificates" - } - }, - { - "name": "logdir", - "hostPath": { - "path": "/var/log" - } - }, - { - "name": "kubeconfig-volume", - "secret": { - "secretName": "kubeconfig" - } - } - ], - "restartPolicy": "Always" - } -} diff --git a/test/kubemark/resources/heapster_template.json b/test/kubemark/resources/heapster_template.json deleted file mode 100644 index 8164ce31740..00000000000 --- a/test/kubemark/resources/heapster_template.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "kind": "ReplicationController", - "apiVersion": "v1", - "metadata": { - "name": "heapster-v1.3.0", - "labels": { - "k8s-app": "heapster", - "version": "v1.3.0" - } - }, - "spec": { - "replicas": 1, - "selector": { - "k8s-app": "heapster", - "version": "v1.3.0" - }, - "template": { - "metadata": { - "labels": { - "k8s-app": "heapster", - "version": "v1.3.0" - } - }, - "spec": { - "volumes": [ - { - "name": "kubeconfig-volume", - "secret": { - "secretName": "kubeconfig" - } - } - ], - "containers": [ - { - "name": "heapster", - "image": "registry.k8s.io/heapster:v1.3.0", - "resources": { - "requests": { - "cpu": "{{METRICS_CPU}}m", - "memory": "{{METRICS_MEM}}Mi" - } - }, - "command": [ - "/heapster" - ], - "args": [ - "--source=kubernetes:https://{{MASTER_IP}}:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/heapster.kubeconfig" - ], - "volumeMounts": [ - { - "name": "kubeconfig-volume", - "mountPath": "/kubeconfig" - } - ] - }, - { - "name": "eventer", - "image": "registry.k8s.io/heapster:v1.3.0", - "resources": { - "requests": { - "memory": "{{EVENTER_MEM}}Ki" - } - }, - "command": [ - "/eventer" - ], - "args": [ - "--source=kubernetes:https://{{MASTER_IP}}:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/heapster.kubeconfig" - ], - "volumeMounts": [ - { - "name": "kubeconfig-volume", - "mountPath": "/kubeconfig" - } - ] - - }] - } - } - } -} - diff --git a/test/kubemark/resources/hollow-node_template.yaml b/test/kubemark/resources/hollow-node_template.yaml deleted file mode 100644 index a6cc19c889d..00000000000 --- a/test/kubemark/resources/hollow-node_template.yaml +++ /dev/null @@ -1,138 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: hollow-node - labels: - name: hollow-node - {{kubemark_mig_config}} -spec: - replicas: {{numreplicas}} - selector: - name: hollow-node - template: - metadata: - labels: - name: hollow-node - {{kubemark_mig_config}} - spec: - initContainers: - - name: init-inotify-limit - image: busybox:1.32 - command: ['sysctl', '-w', 'fs.inotify.max_user_instances=1000'] - securityContext: - privileged: true - volumes: - - name: kubeconfig-volume - secret: - secretName: kubeconfig - - name: kernelmonitorconfig-volume - configMap: - name: node-configmap - - name: logs-volume - hostPath: - path: /var/log - - name: containerd - hostPath: - path: /run/containerd - - name: no-serviceaccount-access-to-real-master - emptyDir: {} - containers: - - name: hollow-kubelet - image: {{kubemark_image_registry}}/kubemark:{{kubemark_image_tag}} - ports: - - containerPort: 4194 - - containerPort: 10250 - - containerPort: 10255 - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - command: [ - "/go-runner", - "-log-file=/var/log/kubelet-$(NODE_NAME).log", - "/kubemark", - "--morph=kubelet", - "--name=$(NODE_NAME)", - "--kubeconfig=/kubeconfig/kubelet.kubeconfig", - "--node-labels={{hollow_node_labels}}", - {{hollow_kubelet_params}} - ] - volumeMounts: - - name: kubeconfig-volume - mountPath: /kubeconfig - readOnly: true - - name: logs-volume - mountPath: /var/log - - name: containerd - mountPath: /run/containerd - resources: - requests: - cpu: {{hollow_kubelet_millicpu}}m - memory: {{hollow_kubelet_mem_Ki}}Ki - securityContext: - privileged: true - - name: hollow-proxy - image: {{kubemark_image_registry}}/kubemark:{{kubemark_image_tag}} - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - command: [ - "/go-runner", - "-log-file=/var/log/kubeproxy-$(NODE_NAME).log", - "/kubemark", - "--morph=proxy", - "--name=$(NODE_NAME)", - "--kubeconfig=/kubeconfig/kubeproxy.kubeconfig", - {{hollow_proxy_params}} - ] - volumeMounts: - - name: kubeconfig-volume - mountPath: /kubeconfig - readOnly: true - - name: logs-volume - mountPath: /var/log - resources: - requests: - cpu: {{hollow_proxy_millicpu}}m - memory: {{hollow_proxy_mem_Ki}}Ki - - name: hollow-node-problem-detector - image: registry.k8s.io/node-problem-detector/node-problem-detector:v1.35.2 - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - command: - - /bin/sh - - -c - - /node-problem-detector --system-log-monitors=/config/kernel.monitor --apiserver-override="https://{{master_ip}}:443?inClusterConfig=false&auth=/kubeconfig/npd.kubeconfig" --alsologtostderr 1>>/var/log/npd-$(NODE_NAME).log 2>&1 - volumeMounts: - - name: kubeconfig-volume - mountPath: /kubeconfig - readOnly: true - - name: kernelmonitorconfig-volume - mountPath: /config - readOnly: true - - name: no-serviceaccount-access-to-real-master - mountPath: /var/run/secrets/kubernetes.io/serviceaccount - readOnly: true - - name: logs-volume - mountPath: /var/log - resources: - requests: - cpu: {{npd_millicpu}}m - memory: {{npd_mem_Ki}}Ki - securityContext: - privileged: true - # Keep the pod running on unreachable node for 15 minutes. - # This time should be sufficient for a VM reboot and should - # avoid recreating a new hollow node. - # See https://github.com/kubernetes/kubernetes/issues/67120 for context. - tolerations: - - key: "node.kubernetes.io/unreachable" - operator: "Exists" - effect: "NoExecute" - tolerationSeconds: 900 diff --git a/test/kubemark/resources/kernel-monitor.json b/test/kubemark/resources/kernel-monitor.json deleted file mode 100644 index 02c47420d77..00000000000 --- a/test/kubemark/resources/kernel-monitor.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "plugin": "filelog", - "pluginConfig": { - "timestamp": "dummy", - "message": "dummy", - "timestampFormat": "dummy" - }, - "logPath": "/dev/null", - "lookback": "10m", - "bufferSize": 10, - "source": "kernel-monitor", - "conditions": [ - { - "type": "KernelDeadlock", - "reason": "KernelHasNoDeadlock", - "message": "kernel has no deadlock" - } - ], - "rules": [] -} diff --git a/test/kubemark/resources/kube_dns_template.yaml b/test/kubemark/resources/kube_dns_template.yaml deleted file mode 100644 index d943d810c74..00000000000 --- a/test/kubemark/resources/kube_dns_template.yaml +++ /dev/null @@ -1,187 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kube-dns - namespace: kubemark - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile - kubernetes.io/name: "KubeDNS" -spec: - selector: - k8s-app: kube-dns - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-dns - namespace: kubemark - labels: - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: kube-dns - namespace: kubemark - labels: - addonmanager.kubernetes.io/mode: EnsureExists ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kube-dns - namespace: kubemark - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - addonmanager.kubernetes.io/mode: Reconcile -spec: - strategy: - rollingUpdate: - maxSurge: 10% - maxUnavailable: 0 - selector: - matchLabels: - k8s-app: kube-dns - template: - metadata: - labels: - k8s-app: kube-dns - spec: - priorityClassName: system-node-critical - tolerations: - - key: "CriticalAddonsOnly" - operator: "Exists" - volumes: - - name: kube-dns-config - configMap: - name: kube-dns - optional: true - - name: secret-volume - secret: - secretName: kubeconfig - containers: - - name: kubedns - image: registry.k8s.io/k8s-dns-kube-dns-amd64:1.14.9 - resources: - limits: - memory: 170Mi - requests: - cpu: 100m - memory: 70Mi - livenessProbe: - httpGet: - path: /healthcheck/kubedns - port: 10054 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - readinessProbe: - httpGet: - path: /readiness - port: 8081 - scheme: HTTP - initialDelaySeconds: 3 - timeoutSeconds: 5 - args: - - --domain={{dns_domain}}. - - --dns-port=10053 - - --config-dir=/kube-dns-config - - --kubecfg-file=/etc/secret-volume/dns.kubeconfig - - --v=2 - env: - - name: PROMETHEUS_PORT - value: "10055" - ports: - - containerPort: 10053 - name: dns-local - protocol: UDP - - containerPort: 10053 - name: dns-tcp-local - protocol: TCP - - containerPort: 10055 - name: metrics - protocol: TCP - volumeMounts: - - name: kube-dns-config - mountPath: /kube-dns-config - - name: secret-volume - mountPath: /etc/secret-volume - - name: dnsmasq - image: registry.k8s.io/k8s-dns-dnsmasq-nanny-amd64:1.14.9 - livenessProbe: - httpGet: - path: /healthcheck/dnsmasq - port: 10054 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - args: - - -v=2 - - -logtostderr - - -configDir=/etc/k8s/dns/dnsmasq-nanny - - -restartDnsmasq=true - - -- - - -k - - --cache-size=1000 - - --no-negcache - - --dns-loop-detect - - --log-facility=- - - --server=/{{dns_domain}}/127.0.0.1#10053 - - --server=/in-addr.arpa/127.0.0.1#10053 - - --server=/ip6.arpa/127.0.0.1#10053 - ports: - - containerPort: 53 - name: dns - protocol: UDP - - containerPort: 53 - name: dns-tcp - protocol: TCP - # see: https://github.com/kubernetes/kubernetes/issues/29055 for details - resources: - requests: - cpu: 150m - memory: 20Mi - volumeMounts: - - name: kube-dns-config - mountPath: /etc/k8s/dns/dnsmasq-nanny - - name: sidecar - image: registry.k8s.io/k8s-dns-sidecar-amd64:1.14.9 - livenessProbe: - httpGet: - path: /metrics - port: 10054 - scheme: HTTP - initialDelaySeconds: 60 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - args: - - --v=2 - - --logtostderr - - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.{{dns_domain}},5,SRV - - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.{{dns_domain}},5,SRV - ports: - - containerPort: 10054 - name: metrics - protocol: TCP - resources: - requests: - memory: 20Mi - cpu: 10m - dnsPolicy: Default # Don't use cluster DNS. - serviceAccountName: kube-dns diff --git a/test/kubemark/resources/kubemark-ns.json b/test/kubemark/resources/kubemark-ns.json deleted file mode 100644 index ea1e887dfba..00000000000 --- a/test/kubemark/resources/kubemark-ns.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "kind": "Namespace", - "apiVersion": "v1", - "metadata": { - "name": "kubemark" - } -} diff --git a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/README.md b/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/README.md deleted file mode 100644 index 1df67db622e..00000000000 --- a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/README.md +++ /dev/null @@ -1 +0,0 @@ -These resources are used to add extra (non-default) bindings to kubemark to match users and groups that are particular to the kubemark environment. These are not standard bootstrap bindings and not standard users they are bound to, and have been adapted from cluster/addons/e2e-rbac-bindings. Tighten/loosen these access rights as required in future. diff --git a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/cluster-autoscaler-binding.yaml b/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/cluster-autoscaler-binding.yaml deleted file mode 100644 index 4e1a89c146c..00000000000 --- a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/cluster-autoscaler-binding.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# This is the role binding for the kubemark cluster autoscaler. -# TODO: Use proper Cluster Autoscaler role (github.com/kubernetes/autoscaler/issues/383) -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cluster-autoscaler-view-binding - labels: - kubernetes.io/cluster-service: "true" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: system:cluster-autoscaler diff --git a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/heapster-binding.yaml b/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/heapster-binding.yaml deleted file mode 100644 index f2feb860ac1..00000000000 --- a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/heapster-binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# This is the role binding for the kubemark heapster. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: heapster-view-binding - labels: - kubernetes.io/cluster-service: "true" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:heapster -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: system:heapster diff --git a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kube-dns-binding.yaml b/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kube-dns-binding.yaml deleted file mode 100644 index 3f9b0d3dff3..00000000000 --- a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kube-dns-binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# This is the role binding for the kubemark kube-dns. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kube-dns-view-binding - labels: - kubernetes.io/cluster-service: "true" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-dns -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: system:kube-dns diff --git a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kubecfg-binding.yaml b/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kubecfg-binding.yaml deleted file mode 100644 index db8ffee35b7..00000000000 --- a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kubecfg-binding.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# This is the role binding for the local kubectl, which is -# used for listing hollow-nodes in start-kubemark.sh and -# send resource creation requests, etc in run-e2e-tests.sh. -# Also useful if you manually want to use local kubectl. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubecfg-cluster-admin - labels: - kubernetes.io/cluster-service: "true" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: kubecfg diff --git a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kubelet-binding.yaml b/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kubelet-binding.yaml deleted file mode 100644 index 2c59627ae36..00000000000 --- a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/kubelet-binding.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# The Kubemark environment currently gives all kubelets a single shared credential. -# -# TODO: give each kubelet a credential in the system:nodes group with username system:node:, -# to exercise the Node authorizer and admission, then remove this binding -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubelet-node - labels: - kubernetes.io/cluster-service: "true" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:node -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: kubelet diff --git a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/npd-binding.yaml b/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/npd-binding.yaml deleted file mode 100644 index ced81a55a99..00000000000 --- a/test/kubemark/resources/manifests/addons/kubemark-rbac-bindings/npd-binding.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# This is the role binding for the node-problem-detector. -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: node-problem-detector-binding - labels: - kubernetes.io/cluster-service: "true" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:node-problem-detector -subjects: -- apiGroup: rbac.authorization.k8s.io - kind: User - name: system:node-problem-detector diff --git a/test/kubemark/resources/manifests/kube-addon-manager.yaml b/test/kubemark/resources/manifests/kube-addon-manager.yaml deleted file mode 100644 index 1d7d87ebf41..00000000000 --- a/test/kubemark/resources/manifests/kube-addon-manager.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: kube-addon-manager - namespace: kube-system - labels: - component: kube-addon-manager -spec: - hostNetwork: true - containers: - - name: kube-addon-manager - image: {{kube_docker_registry}}/addon-manager/kube-addon-manager:v9.1.8 - command: - - /bin/bash - - -c - - /opt/kube-addons-main.sh 1>>/var/log/kube-addon-manager.log 2>&1 - resources: - requests: - cpu: 5m - memory: 50Mi - volumeMounts: - - name: addons - mountPath: /etc/kubernetes/ - readOnly: true - - name: varlog - mountPath: /var/log/kube-addon-manager.log - - mountPath: /etc/srv/kubernetes/addon-manager/ - name: srvkube - readOnly: true - env: - - name: KUBECTL_OPTS - value: '--kubeconfig=/etc/srv/kubernetes/addon-manager/kubeconfig' - volumes: - - name: addons - hostPath: - path: /etc/kubernetes/ - - name: varlog - hostPath: - path: /var/log/kube-addon-manager.log - type: FileOrCreate - - hostPath: - path: /etc/srv/kubernetes/addon-manager/ - name: srvkube diff --git a/test/kubemark/resources/start-kubemark-master.sh b/test/kubemark/resources/start-kubemark-master.sh deleted file mode 100755 index f6507bd4701..00000000000 --- a/test/kubemark/resources/start-kubemark-master.sh +++ /dev/null @@ -1,776 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Script that starts kubelet on kubemark-master as a supervisord process -# and then runs the master components as pods using kubelet. - -set -o errexit -set -o nounset -set -o pipefail - -# Define key path variables. -KUBE_ROOT="/home/kubernetes" -KUBE_BINDIR="${KUBE_ROOT}/kubernetes/server/bin" - -function config-ip-firewall { - echo "Configuring IP firewall rules" - # The GCI image has host firewall which drop most inbound/forwarded packets. - # We need to add rules to accept all TCP/UDP/ICMP packets. - if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then - echo "Add rules to accept all inbound TCP/UDP/ICMP packets" - iptables -A INPUT -w -p TCP -j ACCEPT - iptables -A INPUT -w -p UDP -j ACCEPT - iptables -A INPUT -w -p ICMP -j ACCEPT - fi - if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then - echo "Add rules to accept all forwarded TCP/UDP/ICMP packets" - iptables -A FORWARD -w -p TCP -j ACCEPT - iptables -A FORWARD -w -p UDP -j ACCEPT - iptables -A FORWARD -w -p ICMP -j ACCEPT - fi -} - -function create-dirs { - echo "Creating required directories" - mkdir -p /var/lib/kubelet - mkdir -p /etc/kubernetes/manifests - mkdir -p /etc/kubernetes/addons -} - -# Setup working directory for kubelet. -function setup-kubelet-dir { - echo "Making /var/lib/kubelet executable for kubelet" - mount -B /var/lib/kubelet /var/lib/kubelet/ - mount -B -o remount,exec,suid,dev /var/lib/kubelet -} - -# Remove any default etcd config dirs/files. -function delete-default-etcd-configs { - if [[ -d /etc/etcd ]]; then - rm -rf /etc/etcd - fi - if [[ -e /etc/default/etcd ]]; then - rm -f /etc/default/etcd - fi - if [[ -e /etc/systemd/system/etcd.service ]]; then - rm -f /etc/systemd/system/etcd.service - fi - if [[ -e /etc/init.d/etcd ]]; then - rm -f /etc/init.d/etcd - fi -} - -# Compute etcd related variables. -function compute-etcd-variables { - ETCD_IMAGE="${ETCD_IMAGE:-}" - ETCD_QUOTA_BYTES="" - if [ "${ETCD_VERSION:0:2}" == "3." ]; then - # TODO: Set larger quota to see if that helps with - # 'mvcc: database space exceeded' errors. If so, pipe - # though our setup scripts. - ETCD_QUOTA_BYTES=" --quota-backend-bytes=4294967296 " - fi -} - -# Formats the given device ($1) if needed and mounts it at given mount point -# ($2). -function safe-format-and-mount() { - device=$1 - mountpoint=$2 - - # Format only if the disk is not already formatted. - if ! tune2fs -l "${device}" ; then - echo "Formatting '${device}'" - mkfs.ext4 -F "${device}" - fi - - echo "Mounting '${device}' at '${mountpoint}'" - mount -o discard,defaults "${device}" "${mountpoint}" -} - -# Finds a PD device with name '$1' attached to the master. -function find-attached-pd() { - local -r pd_name=$1 - if [[ ! -e /dev/disk/by-id/${pd_name} ]]; then - echo "" - fi - device_info=$(ls -l "/dev/disk/by-id/${pd_name}") - relative_path=${device_info##* } - echo "/dev/disk/by-id/${relative_path}" -} - -# Mounts a persistent disk (formatting if needed) to store the persistent data -# on the master. safe-format-and-mount only formats an unformatted disk, and -# mkdir -p will leave a directory be if it already exists. -function mount-pd() { - local -r pd_name=$1 - local -r mount_point=$2 - - if [[ -z "${find-attached-pd ${pd_name}}" ]]; then - echo "Can't find ${pd_name}. Skipping mount." - return - fi - - local -r pd_path="/dev/disk/by-id/${pd_name}" - echo "Mounting PD '${pd_path}' at '${mount_point}'" - # Format and mount the disk, create directories on it for all of the master's - # persistent data, and link them to where they're used. - mkdir -p "${mount_point}" - safe-format-and-mount "${pd_path}" "${mount_point}" - echo "Mounted PD '${pd_path}' at '${mount_point}'" - - # NOTE: These locations on the PD store persistent data, so to maintain - # upgradeability, these locations should not change. If they do, take care - # to maintain a migration path from these locations to whatever new - # locations. -} - -# Create kubeconfig for controller-manager's service account authentication. -function create-kubecontrollermanager-kubeconfig { - echo "Creating kube-controller-manager kubeconfig file" - mkdir -p "${KUBE_ROOT}/k8s_auth_data/kube-controller-manager" - cat <"${KUBE_ROOT}/k8s_auth_data/kube-controller-manager/kubeconfig" -apiVersion: v1 -kind: Config -users: -- name: kube-controller-manager - user: - token: ${KUBE_CONTROLLER_MANAGER_TOKEN} -clusters: -- name: local - cluster: - insecure-skip-tls-verify: true - server: https://localhost:443 -contexts: -- context: - cluster: local - user: kube-controller-manager - name: service-account-context -current-context: service-account-context -EOF -} - -function create-kubescheduler-kubeconfig { - echo "Creating kube-scheduler kubeconfig file" - mkdir -p "${KUBE_ROOT}/k8s_auth_data/kube-scheduler" - cat <"${KUBE_ROOT}/k8s_auth_data/kube-scheduler/kubeconfig" -apiVersion: v1 -kind: Config -users: -- name: kube-scheduler - user: - token: ${KUBE_SCHEDULER_TOKEN} -clusters: -- name: local - cluster: - insecure-skip-tls-verify: true - server: https://localhost:443 -contexts: -- context: - cluster: local - user: kube-scheduler - name: kube-scheduler -current-context: kube-scheduler -EOF -} - -function create-addonmanager-kubeconfig { - echo "Creating addonmanager kubeconfig file" - mkdir -p "${KUBE_ROOT}/k8s_auth_data/addon-manager" - cat <"${KUBE_ROOT}/k8s_auth_data/addon-manager/kubeconfig" -apiVersion: v1 -kind: Config -users: -- name: addon-manager - user: - token: ${ADDON_MANAGER_TOKEN} -clusters: -- name: local - cluster: - insecure-skip-tls-verify: true - server: https://localhost:443 -contexts: -- context: - cluster: local - user: addon-manager - name: addon-manager -current-context: addon-manager -EOF -} - -function assemble-docker-flags { - echo "Assemble docker command line flags" - local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false" - docker_opts+=" --log-level=debug" # Since it's a test cluster - # TODO(shyamjvs): Incorporate network plugin options, etc later. - echo "DOCKER_OPTS=\"${docker_opts}\"" > /etc/default/docker - echo "DOCKER_NOFILE=65536" >> /etc/default/docker # For setting ulimit -n - systemctl restart docker -} - -# A helper function for loading a docker image. It keeps trying up to 5 times. -# -# $1: Full path of the docker image -function try-load-docker-image { - local -r img=$1 - echo "Try to load docker image file ${img}" - # Temporarily turn off errexit, because we don't want to exit on first failure. - set +e - local -r max_attempts=5 - local -i attempt_num=1 - until timeout 30 docker load -i "${img}"; do - if [[ "${attempt_num}" == "${max_attempts}" ]]; then - echo "Fail to load docker image file ${img} after ${max_attempts} retries. Exit!!" - exit 1 - else - attempt_num=$((attempt_num+1)) - sleep 5 - fi - done - # Re-enable errexit. - set -e -} - -# Loads kube-system docker images. It is better to do it before starting kubelet, -# as kubelet will restart docker daemon, which may interfere with loading images. -function load-docker-images { - echo "Start loading kube-system docker images" - local -r img_dir="${KUBE_BINDIR}" - try-load-docker-image "${img_dir}/kube-apiserver.tar" - try-load-docker-image "${img_dir}/kube-controller-manager.tar" - try-load-docker-image "${img_dir}/kube-scheduler.tar" -} - -# Computes command line arguments to be passed to kubelet. -function compute-kubelet-params { - local params="${KUBELET_TEST_ARGS:-}" - params+=" --cgroup-root=/" - params+=" --cloud-provider=gce" - params+=" --pod-manifest-path=/etc/kubernetes/manifests" - if [[ -n "${KUBELET_PORT:-}" ]]; then - params+=" --port=${KUBELET_PORT}" - fi - params+=" --enable-debugging-handlers=false" - params+=" --hairpin-mode=none" - echo "${params}" -} - -# Creates the systemd config file for kubelet.service. -function create-kubelet-conf() { - local -r kubelet_bin="$1" - local -r kubelet_env_file="/etc/default/kubelet" - local -r flags=$(compute-kubelet-params) - echo "KUBELET_OPTS=\"${flags}\"" > "${kubelet_env_file}" - - # Write the systemd service file for kubelet. - cat </etc/systemd/system/kubelet.service -[Unit] -Description=Kubermark kubelet -Requires=network-online.target -After=network-online.target - -[Service] -Restart=always -RestartSec=10 -EnvironmentFile=${kubelet_env_file} -ExecStart=${kubelet_bin} \$KUBELET_OPTS - -[Install] -WantedBy=multi-user.target -EOF -} - -# This function assembles the kubelet systemd service file and starts it using -# systemctl, on the kubemark master. -function start-kubelet { - # Create systemd config. - local -r kubelet_bin="/usr/bin/kubelet" - create-kubelet-conf "${kubelet_bin}" - - # Flush iptables nat table - iptables -t nat -F || true - - # Start the kubelet service. - systemctl start kubelet.service -} - -# Create the log file and set its properties. -# -# $1 is the file to create. -function prepare-log-file { - touch "$1" - chmod 644 "$1" - chown root:root "$1" -} - -# A helper function for copying addon manifests and set dir/files -# permissions. -# -# $1: addon category under /etc/kubernetes -# $2: manifest source dir -function setup-addon-manifests { - local -r src_dir="${KUBE_ROOT}/$2" - local -r dst_dir="/etc/kubernetes/$1/$2" - - if [[ ! -d "${dst_dir}" ]]; then - mkdir -p "${dst_dir}" - fi - - local files - files=$(find "${src_dir}" -maxdepth 1 -name "*.yaml") - if [[ -n "${files}" ]]; then - cp "${src_dir}/"*.yaml "${dst_dir}" - fi - chown -R root:root "${dst_dir}" - chmod 755 "${dst_dir}" - chmod 644 "${dst_dir}"/* -} - -# Write the config for the audit policy. -# Note: This duplicates the function in cluster/gce/gci/configure-helper.sh. -# TODO: Get rid of this function when #53321 is fixed. -function create-master-audit-policy { - local -r path="${1}" - local -r policy="${2:-}" - - if [[ -n "${policy}" ]]; then - echo "${policy}" > "${path}" - return - fi - - # Known api groups - local -r known_apis=' - - group: "" # core - - group: "admissionregistration.k8s.io" - - group: "apiextensions.k8s.io" - - group: "apiregistration.k8s.io" - - group: "apps" - - group: "authentication.k8s.io" - - group: "authorization.k8s.io" - - group: "autoscaling" - - group: "batch" - - group: "certificates.k8s.io" - - group: "extensions" - - group: "metrics" - - group: "networking.k8s.io" - - group: "policy" - - group: "rbac.authorization.k8s.io" - - group: "storage.k8s.io"' - - cat <"${path}" -apiVersion: audit.k8s.io/v1 -kind: Policy -rules: - # The following requests were manually identified as high-volume and low-risk, - # so drop them. - - level: None - users: ["system:kube-proxy"] - verbs: ["watch"] - resources: - - group: "" # core - resources: ["endpoints", "services", "services/status"] - - level: None - # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port. - # TODO(#46983): Change this to the ingress controller service account. - users: ["system:unsecured"] - namespaces: ["kube-system"] - verbs: ["get"] - resources: - - group: "" # core - resources: ["configmaps"] - - level: None - users: ["kubelet"] # legacy kubelet identity - verbs: ["get"] - resources: - - group: "" # core - resources: ["nodes", "nodes/status"] - - level: None - userGroups: ["system:nodes"] - verbs: ["get"] - resources: - - group: "" # core - resources: ["nodes", "nodes/status"] - - level: None - users: - - system:kube-controller-manager - - system:kube-scheduler - - system:serviceaccount:kube-system:endpoint-controller - verbs: ["get", "update"] - namespaces: ["kube-system"] - resources: - - group: "" # core - resources: ["endpoints"] - - level: None - users: ["system:apiserver"] - verbs: ["get"] - resources: - - group: "" # core - resources: ["namespaces", "namespaces/status", "namespaces/finalize"] - # Don't log HPA fetching metrics. - - level: None - users: - - system:kube-controller-manager - verbs: ["get", "list"] - resources: - - group: "metrics" - # Don't log these read-only URLs. - - level: None - nonResourceURLs: - - /healthz* - - /version - - /swagger* - # Don't log events requests. - - level: None - resources: - - group: "" # core - resources: ["events"] - # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes - - level: Request - users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"] - verbs: ["update","patch"] - resources: - - group: "" # core - resources: ["nodes/status", "pods/status"] - omitStages: - - "RequestReceived" - - level: Request - userGroups: ["system:nodes"] - verbs: ["update","patch"] - resources: - - group: "" # core - resources: ["nodes/status", "pods/status"] - omitStages: - - "RequestReceived" - # deletecollection calls can be large, don't log responses for expected namespace deletions - - level: Request - users: ["system:serviceaccount:kube-system:namespace-controller"] - verbs: ["deletecollection"] - omitStages: - - "RequestReceived" - # Secrets, ConfigMaps, TokenRequest and TokenReviews can contain sensitive & binary data, - # so only log at the Metadata level. - - level: Metadata - resources: - - group: "" # core - resources: ["secrets", "configmaps", "serviceaccounts/token"] - - group: authentication.k8s.io - resources: ["tokenreviews"] - omitStages: - - "RequestReceived" - # Get responses can be large; skip them. - - level: Request - verbs: ["get", "list", "watch"] - resources: ${known_apis} - omitStages: - - "RequestReceived" - # Default level for known APIs - - level: RequestResponse - resources: ${known_apis} - omitStages: - - "RequestReceived" - # Default level for all other requests. - - level: Metadata - omitStages: - - "RequestReceived" -EOF -} - -# Computes command line arguments to be passed to etcd. -function compute-etcd-params { - local params="${ETCD_TEST_ARGS:-}" - params+=" --name=etcd-$(hostname -s)" - params+=" --listen-peer-urls=http://127.0.0.1:2380" - params+=" --advertise-client-urls=http://127.0.0.1:2379" - params+=" --listen-client-urls=http://0.0.0.0:2379" - - # Enable apiserver->etcd auth. - params+=" --client-cert-auth" - params+=" --trusted-ca-file /etc/srv/kubernetes/etcd-apiserver-ca.crt" - params+=" --cert-file /etc/srv/kubernetes/etcd-apiserver-server.crt" - params+=" --key-file /etc/srv/kubernetes/etcd-apiserver-server.key" - - params+=" --data-dir=/var/etcd/data" - params+=" ${ETCD_QUOTA_BYTES}" - echo "${params}" -} - -# Computes command line arguments to be passed to etcd-events. -function compute-etcd-events-params { - local params="${ETCD_TEST_ARGS:-}" - params+=" --name=etcd-$(hostname -s)" - params+=" --listen-peer-urls=http://127.0.0.1:2381" - params+=" --advertise-client-urls=http://127.0.0.1:4002" - params+=" --listen-client-urls=http://0.0.0.0:4002" - params+=" --data-dir=/var/etcd/data-events" - params+=" ${ETCD_QUOTA_BYTES}" - echo "${params}" -} - -# Computes command line arguments to be passed to apiserver. -function compute-kube-apiserver-params { - local params="--insecure-bind-address=0.0.0.0" - params+=" --etcd-servers=${ETCD_SERVERS:-http://127.0.0.1:2379}" - if [[ -z "${ETCD_SERVERS:-}" ]]; then - params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#${EVENT_STORE_URL}}" - elif [[ -n "${ETCD_SERVERS_OVERRIDES:-}" ]]; then - params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}" - fi - # Enable apiserver->etcd auth. - params+=" --etcd-cafile=/etc/srv/kubernetes/etcd-apiserver-ca.crt" - params+=" --etcd-certfile=/etc/srv/kubernetes/etcd-apiserver-client.crt" - params+=" --etcd-keyfile=/etc/srv/kubernetes/etcd-apiserver-client.key" - - params+=" --tls-cert-file=/etc/srv/kubernetes/server.cert" - params+=" --tls-private-key-file=/etc/srv/kubernetes/server.key" - params+=" --requestheader-client-ca-file=/etc/srv/kubernetes/aggr_ca.crt" - params+=" --requestheader-allowed-names=aggregator" - params+=" --requestheader-extra-headers-prefix=X-Remote-Extra-" - params+=" --requestheader-group-headers=X-Remote-Group" - params+=" --requestheader-username-headers=X-Remote-User" - params+=" --proxy-client-cert-file=/etc/srv/kubernetes/proxy_client.crt" - params+=" --proxy-client-key-file=/etc/srv/kubernetes/proxy_client.key" - params+=" --enable-aggregator-routing=true" - params+=" --client-ca-file=/etc/srv/kubernetes/ca.crt" - params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv" - params+=" --secure-port=443" - params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" - params+=" --admission-control=${CUSTOM_ADMISSION_PLUGINS}" - params+=" --authorization-mode=Node,RBAC" - params+=" --allow-privileged=true" - if [[ -n "${STORAGE_BACKEND:-}" ]]; then - params+=" --storage-backend=${STORAGE_BACKEND}" - fi - if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then - params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}" - fi - if [[ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]]; then - params+=" --etcd-compaction-interval=${ETCD_COMPACTION_INTERVAL_SEC}s" - fi - if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then - params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}" - fi - if [[ "${NUM_NODES}" -ge 3000 ]]; then - params+=" --max-requests-inflight=3000 --max-mutating-requests-inflight=1000" - elif [[ "${NUM_NODES}" -ge 1000 ]]; then - params+=" --max-requests-inflight=1500 --max-mutating-requests-inflight=500" - fi - if [[ -n "${RUNTIME_CONFIG:-}" ]]; then - params+=" --runtime-config=${RUNTIME_CONFIG}" - fi - if [[ -n "${FEATURE_GATES:-}" ]]; then - params+=" --feature-gates=${FEATURE_GATES}" - fi - if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then - # Create the audit policy file, and mount it into the apiserver pod. - create-master-audit-policy "${audit_policy_file}" "${ADVANCED_AUDIT_POLICY:-}" - - # The config below matches the one in cluster/gce/gci/configure-helper.sh. - # TODO: Currently supporting just log backend. Support webhook if needed. - params+=" --audit-policy-file=${audit_policy_file}" - params+=" --audit-log-path=/var/log/kube-apiserver-audit.log" - params+=" --audit-log-maxage=0" - params+=" --audit-log-maxbackup=0" - params+=" --audit-log-maxsize=2000000000" - fi - # Append APISERVER_TEST_ARGS to the end, which will allow for - # the above defaults to be overridden. - params+=" ${APISERVER_TEST_ARGS:-}" - echo "${params}" -} - -# Computes command line arguments to be passed to controller-manager. -function compute-kube-controller-manager-params { - local params="${CONTROLLER_MANAGER_TEST_ARGS:-}" - params+=" --use-service-account-credentials" - params+=" --kubeconfig=/etc/srv/kubernetes/kube-controller-manager/kubeconfig" - params+=" --service-account-private-key-file=/etc/srv/kubernetes/server.key" - params+=" --root-ca-file=/etc/srv/kubernetes/ca.crt" - params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}" - params+=" --cluster-cidr=${CLUSTER_IP_RANGE}" - params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}" - params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}" - echo "${params}" -} - -# Computes command line arguments to be passed to scheduler. -function compute-kube-scheduler-params { - local params="${SCHEDULER_TEST_ARGS:-}" - params+=" --kubeconfig=/etc/srv/kubernetes/kube-scheduler/kubeconfig" - echo "${params}" -} - -# Computes command line arguments to be passed to addon-manager. -function compute-kube-addon-manager-params { - echo "" -} - -# Start a kubernetes master component '$1' which can be any of the following: -# 1. etcd -# 2. etcd-events -# 3. kube-apiserver -# 4. kube-controller-manager -# 5. kube-scheduler -# 6. kube-addon-manager -# -# It prepares the log file, loads the docker tag, calculates variables, sets them -# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests. -# -# Assumed vars: -# DOCKER_REGISTRY -function start-kubemaster-component() { - echo "Start master component $1" - local -r component=$1 - prepare-log-file /var/log/"${component}".log - local -r src_file="${KUBE_ROOT}/${component}.yaml" - local -r params=$("compute-${component}-params") - - # Evaluate variables. - sed -i -e "s@{{params}}@${params}@g" "${src_file}" - sed -i -e "s@{{kube_docker_registry}}@${DOCKER_REGISTRY}@g" "${src_file}" - sed -i -e "s@{{instance_prefix}}@${INSTANCE_PREFIX}@g" "${src_file}" - if [ "${component:0:4}" == "etcd" ]; then - sed -i -e "s@{{etcd_image}}@${ETCD_IMAGE}@g" "${src_file}" - elif [ "${component}" == "kube-addon-manager" ]; then - setup-addon-manifests "addons" "kubemark-rbac-bindings" - else - local -r component_docker_tag=$(cat "${KUBE_BINDIR}/${component}.docker_tag") - sed -i -e "s@{{${component}_docker_tag}}@${component_docker_tag}@g" "${src_file}" - if [ "${component}" == "kube-apiserver" ]; then - local audit_policy_config_mount="" - local audit_policy_config_volume="" - if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then - read -r -d '' audit_policy_config_mount << EOF -- name: auditpolicyconfigmount - mountPath: ${audit_policy_file} - readOnly: true -EOF - read -r -d '' audit_policy_config_volume << EOF -- name: auditpolicyconfigmount - hostPath: - path: ${audit_policy_file} - type: FileOrCreate -EOF - fi - sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}" - sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}" - fi - fi - cp "${src_file}" /etc/kubernetes/manifests -} - -############################### Main Function ######################################## -echo "Start to configure master instance for kubemark" - -# Extract files from the server tar and setup master env variables. -cd "${KUBE_ROOT}" -if [[ ! -d "${KUBE_ROOT}/kubernetes" ]]; then - tar xzf kubernetes-server-linux-amd64.tar.gz -fi -source "${KUBE_ROOT}/kubemark-master-env.sh" - -# Setup IP firewall rules, required directory structure and etcd config. -config-ip-firewall -create-dirs -setup-kubelet-dir -delete-default-etcd-configs -compute-etcd-variables - -# Setup authentication tokens and kubeconfigs for kube-controller-manager and kube-scheduler, -# only if their kubeconfigs don't already exist as this script could be running on reboot. -if [[ ! -f "${KUBE_ROOT}/k8s_auth_data/kube-controller-manager/kubeconfig" ]]; then - KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | LC_CTYPE=C tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - echo "${KUBE_CONTROLLER_MANAGER_TOKEN},system:kube-controller-manager,uid:system:kube-controller-manager" >> "${KUBE_ROOT}/k8s_auth_data/known_tokens.csv" - create-kubecontrollermanager-kubeconfig -fi -if [[ ! -f "${KUBE_ROOT}/k8s_auth_data/kube-scheduler/kubeconfig" ]]; then - KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | LC_CTYPE=C tr -d "=+/" | dd bs=32 count=1 2>/dev/null) - echo "${KUBE_SCHEDULER_TOKEN},system:kube-scheduler,uid:system:kube-scheduler" >> "${KUBE_ROOT}/k8s_auth_data/known_tokens.csv" - create-kubescheduler-kubeconfig -fi - -ADDON_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | LC_CTYPE=C tr -d "=+/" | dd bs=32 count=1 2>/dev/null) -echo "${ADDON_MANAGER_TOKEN},system:addon-manager,admin,system:masters" >> "${KUBE_ROOT}/k8s_auth_data/known_tokens.csv" -create-addonmanager-kubeconfig - -# Mount master PD for etcd and create symbolic links to it. -{ - main_etcd_mount_point="/mnt/disks/master-pd" - mount-pd "google-master-pd" "${main_etcd_mount_point}" - # Contains all the data stored in etcd. - mkdir -p "${main_etcd_mount_point}/var/etcd" - chmod 700 "${main_etcd_mount_point}/var/etcd" - ln -s -f "${main_etcd_mount_point}/var/etcd" /var/etcd - mkdir -p /etc/srv - # Setup the dynamically generated apiserver auth certs and keys to pd. - mkdir -p "${main_etcd_mount_point}/srv/kubernetes" - ln -s -f "${main_etcd_mount_point}/srv/kubernetes" /etc/srv/kubernetes - # Copy the files to the PD only if they don't exist (so we do it only the first time). - if [[ "$(ls -A ${main_etcd_mount_point}/srv/kubernetes/)" == "" ]]; then - cp -r "${KUBE_ROOT}"/k8s_auth_data/* "${main_etcd_mount_point}/srv/kubernetes/" - fi - # Directory for kube-apiserver to store SSH key (if necessary). - mkdir -p "${main_etcd_mount_point}/srv/sshproxy" - ln -s -f "${main_etcd_mount_point}/srv/sshproxy" /etc/srv/sshproxy -} - -# Mount master PD for event-etcd (if required) and create symbolic links to it. -{ - EVENT_STORE_IP="${EVENT_STORE_IP:-127.0.0.1}" - EVENT_STORE_URL="${EVENT_STORE_URL:-http://${EVENT_STORE_IP}:4002}" - if [ "${EVENT_PD:-}" == "true" ]; then - event_etcd_mount_point="/mnt/disks/master-event-pd" - mount-pd "google-master-event-pd" "${event_etcd_mount_point}" - # Contains all the data stored in event etcd. - mkdir -p "${event_etcd_mount_point}/var/etcd/events" - chmod 700 "${event_etcd_mount_point}/var/etcd/events" - ln -s -f "${event_etcd_mount_point}/var/etcd/events" /var/etcd/events - fi -} - -# Setup docker flags and load images of the master components. -assemble-docker-flags -DOCKER_REGISTRY="registry.k8s.io" -load-docker-images - -readonly audit_policy_file="/etc/audit_policy.config" - -# Start kubelet as a supervisord process and master components as pods. -start-kubelet -if [[ -z "${ETCD_SERVERS:-}" ]]; then - start-kubemaster-component "etcd" - if [ "${EVENT_STORE_IP:-}" == "127.0.0.1" ]; then - start-kubemaster-component "etcd-events" - fi -fi -start-kubemaster-component "kube-apiserver" -start-kubemaster-component "kube-controller-manager" -start-kubemaster-component "kube-scheduler" -start-kubemaster-component "kube-addon-manager" - -# Wait till apiserver is working fine or timeout. -echo -n "Waiting for apiserver to be healthy" -start=$(date +%s) -until [ "$(curl 127.0.0.1:8080/healthz 2> /dev/null)" == "ok" ]; do - echo -n "." - sleep 1 - now=$(date +%s) - if [ $((now - start)) -gt 300 ]; then - echo "Timeout!" - exit 1 - fi -done - -echo "Done for the configuration for kubemark master" diff --git a/test/kubemark/run-e2e-tests.sh b/test/kubemark/run-e2e-tests.sh deleted file mode 100755 index 50ef835a9b4..00000000000 --- a/test/kubemark/run-e2e-tests.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -export KUBERNETES_PROVIDER="kubemark" -export KUBE_CONFIG_FILE="config-default.sh" - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. - -# We need an absolute path to KUBE_ROOT -ABSOLUTE_ROOT=$(readlink -f "${KUBE_ROOT}") - -source "${KUBE_ROOT}/cluster/kubemark/util.sh" - -echo "Kubemark master name: ${MASTER_NAME}" -echo "Kubemark master ip: ${KUBE_MASTER_IP}" - -export KUBE_MASTER_URL="https://${KUBE_MASTER_IP}" -export KUBECONFIG="${ABSOLUTE_ROOT}/test/kubemark/resources/kubeconfig.kubemark" -export E2E_MIN_STARTUP_PODS=0 - -ARGS=("$@") - -if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER}" == "true" ]]; then - ARGS+=("--kubemark-external-kubeconfig=${DEFAULT_KUBECONFIG}") -fi - -# Running locally. -for ((i=0; i < ${#ARGS[@]}; i++)); do - ARGS[i]="$(echo "${ARGS[$i]}" | sed -e 's/\[/\\\[/g' -e 's/\]/\\\]/g' )" -done -"${KUBE_ROOT}/hack/ginkgo-e2e.sh" "--e2e-verify-service-account=false" "--dump-logs-on-failure=false" "${ARGS[@]}" diff --git a/test/kubemark/skeleton/util.sh b/test/kubemark/skeleton/util.sh deleted file mode 100644 index 6e8f4e21f2b..00000000000 --- a/test/kubemark/skeleton/util.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2017 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script contains the helper functions that each provider hosting -# Kubermark must implement to use test/kubemark/start-kubemark.sh and -# test/kubemark/stop-kubemark.sh scripts. - -# This function should authenticate docker to be able to read/write to -# the right container registry (needed for pushing kubemark image). -function authenticate-docker { - echo "Configuring registry authentication" 1>&2 -} - -# This function should create kubemark master and write kubeconfig to -# "${RESOURCE_DIRECTORY}/kubeconfig.kubemark". -# If a cluster uses private master IP, create-kubemark-master might also write -# a second kubeconfig to "${RESOURCE_DIRECTORY}/kubeconfig-internal.kubemark". -# The difference between these two kubeconfigs is that the internal one uses -# private master IP, which might be better suited for setting up hollow nodes. -function create-kubemark-master { - echo "Creating cluster..." -} - -# This function should delete kubemark master. -function delete-kubemark-master { - echo "Deleting cluster..." -} - -# This function should return node labels. -function calculate-node-labels { - echo "" -} - -# Common colors used throughout the kubemark scripts -if [[ -z "${color_start-}" ]]; then - declare -r color_start="\033[" - # shellcheck disable=SC2034 - declare -r color_red="${color_start}0;31m" - # shellcheck disable=SC2034 - declare -r color_yellow="${color_start}0;33m" - # shellcheck disable=SC2034 - declare -r color_green="${color_start}0;32m" - # shellcheck disable=SC2034 - declare -r color_blue="${color_start}1;34m" - # shellcheck disable=SC2034 - declare -r color_cyan="${color_start}1;36m" - # shellcheck disable=SC2034 - declare -r color_norm="${color_start}0m" -fi diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh deleted file mode 100755 index 09e8f21e19e..00000000000 --- a/test/kubemark/start-kubemark.sh +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Script that creates a Kubemark cluster for any given cloud provider. - -set -o errexit -set -o nounset -set -o pipefail - -TMP_ROOT="$(dirname "${BASH_SOURCE[@]}")/../.." -KUBE_ROOT=$(readlink -e "${TMP_ROOT}" 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' "${TMP_ROOT}") - -source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh" -source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh" -source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh" -source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh" - -if [[ -f "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/startup.sh" ]] ; then - source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/startup.sh" -fi - -source "${KUBE_ROOT}/cluster/kubemark/util.sh" - -KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" -KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" -RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" -LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark" -INTERNAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig-internal.kubemark" - -# Generate a random 6-digit alphanumeric tag for the kubemark image. -# Used to uniquify image builds across different invocations of this script. -KUBEMARK_IMAGE_TAG=$(head /dev/urandom | LC_CTYPE=C tr -dc 'a-z0-9' | fold -w 6 | head -n 1) - -# Create a docker image for hollow-node and upload it to the appropriate docker registry. -function create-and-upload-hollow-node-image { - authenticate-docker - KUBEMARK_IMAGE_REGISTRY="${KUBEMARK_IMAGE_REGISTRY:-${CONTAINER_REGISTRY}/${PROJECT}}" - # Build+push the image through makefile. - build_cmd=("make" "${KUBEMARK_IMAGE_MAKE_TARGET}") - MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark" - KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)" - if [[ -z "${KUBEMARK_BIN}" ]]; then - echo 'Cannot find cmd/kubemark binary' - exit 1 - fi - echo "Copying kubemark binary to ${MAKE_DIR}" - cp "${KUBEMARK_BIN}" "${MAKE_DIR}" - CURR_DIR=$(pwd) - cd "${MAKE_DIR}" - REGISTRY=${KUBEMARK_IMAGE_REGISTRY} IMAGE_TAG=${KUBEMARK_IMAGE_TAG} run-cmd-with-retries "${build_cmd[@]}" - rm kubemark - cd "$CURR_DIR" - echo "Created and uploaded the kubemark hollow-node image to docker registry." - # Cleanup the kubemark image after the script exits. - if [[ "${CLEANUP_KUBEMARK_IMAGE:-}" == "true" ]]; then - trap delete-kubemark-image EXIT - fi -} - -function delete-kubemark-image { - delete-image "${KUBEMARK_IMAGE_REGISTRY}/kubemark:${KUBEMARK_IMAGE_TAG}" -} - -# Generate secret and configMap for the hollow-node pods to work, prepare -# manifests of the hollow-node and heapster replication controllers from -# templates, and finally create these resources through kubectl. -function create-kube-hollow-node-resources { - # Create kubemark namespace. - "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" - - # Create configmap for configuring hollow- kubelet, proxy and npd. - "${KUBECTL}" create configmap "node-configmap" --namespace="kubemark" \ - --from-file=kernel.monitor="${RESOURCE_DIRECTORY}/kernel-monitor.json" - - # Create secret for passing kubeconfigs to kubelet, kubeproxy and npd. - # It's bad that all component shares the same kubeconfig. - # TODO(https://github.com/kubernetes/kubernetes/issues/79883): Migrate all components to separate credentials. - "${KUBECTL}" create secret generic "kubeconfig" --type=Opaque --namespace="kubemark" \ - --from-file=kubelet.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \ - --from-file=kubeproxy.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \ - --from-file=npd.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \ - --from-file=heapster.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \ - --from-file=cluster_autoscaler.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \ - --from-file=dns.kubeconfig="${HOLLOWNODE_KUBECONFIG}" - - # Create addon pods. - # Heapster. - mkdir -p "${RESOURCE_DIRECTORY}/addons" - sed "s@{{MASTER_IP}}@${MASTER_IP}@g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json" - metrics_mem_per_node=4 - metrics_mem=$((200 + metrics_mem_per_node*NUM_NODES)) - sed -i'' -e "s@{{METRICS_MEM}}@${metrics_mem}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json" - metrics_cpu_per_node_numerator=${NUM_NODES} - metrics_cpu_per_node_denominator=2 - metrics_cpu=$((80 + metrics_cpu_per_node_numerator / metrics_cpu_per_node_denominator)) - sed -i'' -e "s@{{METRICS_CPU}}@${metrics_cpu}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json" - eventer_mem_per_node=500 - eventer_mem=$((200 * 1024 + eventer_mem_per_node*NUM_NODES)) - sed -i'' -e "s@{{EVENTER_MEM}}@${eventer_mem}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json" - - # Cluster Autoscaler. - if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-}" == "true" ]]; then - echo "Setting up Cluster Autoscaler" - KUBEMARK_AUTOSCALER_MIG_NAME="${KUBEMARK_AUTOSCALER_MIG_NAME:-${NODE_INSTANCE_PREFIX}-group}" - KUBEMARK_AUTOSCALER_MIN_NODES="${KUBEMARK_AUTOSCALER_MIN_NODES:-0}" - KUBEMARK_AUTOSCALER_MAX_NODES="${KUBEMARK_AUTOSCALER_MAX_NODES:-10}" - NUM_NODES=${KUBEMARK_AUTOSCALER_MAX_NODES} - echo "Setting maximum cluster size to ${NUM_NODES}." - KUBEMARK_MIG_CONFIG="autoscaling.k8s.io/nodegroup: ${KUBEMARK_AUTOSCALER_MIG_NAME}" - sed "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/cluster-autoscaler_template.json" > "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json" - sed -i'' -e "s@{{kubemark_autoscaler_mig_name}}@${KUBEMARK_AUTOSCALER_MIG_NAME}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json" - sed -i'' -e "s@{{kubemark_autoscaler_min_nodes}}@${KUBEMARK_AUTOSCALER_MIN_NODES}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json" - sed -i'' -e "s@{{kubemark_autoscaler_max_nodes}}@${KUBEMARK_AUTOSCALER_MAX_NODES}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json" - fi - - # Kube DNS. - if [[ "${ENABLE_KUBEMARK_KUBE_DNS:-}" == "true" ]]; then - echo "Setting up kube-dns" - sed "s@{{dns_domain}}@${KUBE_DNS_DOMAIN}@g" "${RESOURCE_DIRECTORY}/kube_dns_template.yaml" > "${RESOURCE_DIRECTORY}/addons/kube_dns.yaml" - fi - - "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" - - # Create the replication controller for hollow-nodes. - # We allow to override the NUM_REPLICAS when running Cluster Autoscaler. - NUM_REPLICAS=${NUM_REPLICAS:-${NUM_NODES}} - sed "s@{{numreplicas}}@${NUM_REPLICAS}@g" "${RESOURCE_DIRECTORY}/hollow-node_template.yaml" > "${RESOURCE_DIRECTORY}/hollow-node.yaml" - proxy_cpu=20 - if [ "${NUM_NODES}" -gt 1000 ]; then - proxy_cpu=50 - fi - proxy_cpu=${KUBEMARK_HOLLOW_PROXY_MILLICPU:-$proxy_cpu} - proxy_mem_per_node=${KUBEMARK_HOLLOW_PROXY_MEM_PER_NODE_KB:-50} - proxy_mem=$((100 * 1024 + proxy_mem_per_node*NUM_NODES)) - hollow_node_labels=${HOLLOW_NODE_LABELS:-$(calculate-node-labels)} - hollow_kubelet_params=$(eval "for param in ${HOLLOW_KUBELET_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done") - hollow_kubelet_params=${hollow_kubelet_params%?} - hollow_proxy_params=$(eval "for param in ${HOLLOW_PROXY_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done") - hollow_proxy_params=${hollow_proxy_params%?} - - sed -i'' -e "s@{{hollow_kubelet_millicpu}}@${KUBEMARK_HOLLOW_KUBELET_MILLICPU:-40}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s@{{hollow_kubelet_mem_Ki}}@${KUBEMARK_HOLLOW_KUBELET_MEM_KB:-$((100*1024))}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s@{{hollow_proxy_millicpu}}@${proxy_cpu}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s@{{hollow_proxy_mem_Ki}}@${proxy_mem}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s@{{npd_millicpu}}@${KUBEMARK_NPD_MILLICPU:-20}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s@{{npd_mem_Ki}}@${KUBEMARK_NPD_MEM_KB:-$((20*1024))}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s@{{kubemark_image_registry}}@${KUBEMARK_IMAGE_REGISTRY}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s@{{kubemark_image_tag}}@${KUBEMARK_IMAGE_TAG}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s@{{master_ip}}@${MASTER_IP}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s@{{hollow_node_labels}}@${hollow_node_labels}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s@{{hollow_kubelet_params}}@${hollow_kubelet_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s@{{hollow_proxy_params}}@${hollow_proxy_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - sed -i'' -e "s@{{kubemark_mig_config}}@${KUBEMARK_MIG_CONFIG:-}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml" - "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark" - - echo "Created secrets, configMaps, replication-controllers required for hollow-nodes." -} - -# Wait until all hollow-nodes are running or there is a timeout. -function wait-for-hollow-nodes-to-run-or-timeout { - echo -n "Waiting for all hollow-nodes to become Running" - start=$(date +%s) - nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true - ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1)) - - until [[ "${ready}" -ge "${NUM_REPLICAS}" ]]; do - echo -n "." - sleep 1 - now=$(date +%s) - # Fail it if it already took more than 30 minutes. - if [ $((now - start)) -gt 1800 ]; then - echo "" - "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" describe node - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_red} Timeout waiting for all hollow-nodes to become Running. ${color_norm}" - # Try listing nodes again - if it fails it means that API server is not responding - if "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node &> /dev/null; then - echo "Found only ${ready} ready hollow-nodes while waiting for ${NUM_NODES}." - else - echo "Got error while trying to list hollow-nodes. Probably API server is down." - fi - pods=$("${KUBECTL}" get pods -l name=hollow-node --namespace=kubemark) || true - running=$(($(echo "${pods}" | grep -c "Running"))) - echo "${running} hollow-nodes are reported as 'Running'" - not_running=$(($(echo "${pods}" | grep -vc "Running") - 1)) - echo "${not_running} hollow-nodes are reported as NOT 'Running'" - echo "${pods}" | grep -v Running - exit 1 - fi - nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true - ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1)) - done - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_green} Done!${color_norm}" -} - -############################### Main Function ######################################## - -# Setup for hollow-nodes. -function start-hollow-nodes { - # shellcheck disable=SC2154 # Color defined in sourced script - echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}" - create-and-upload-hollow-node-image - create-kube-hollow-node-resources - wait-for-hollow-nodes-to-run-or-timeout -} - -# Annotates the node objects in the kubemark cluster to make their size -# similar to regular nodes. -# TODO(#90833): Replace this with image preloading from ClusterLoader to better -# reflect the reality in kubemark tests. -function resize-node-objects { - if [[ -z "$KUBEMARK_NODE_OBJECT_SIZE_BYTES" ]]; then - return 0 - fi - - annotation_size_bytes="${KUBEMARK_NODE_OBJECT_SIZE_BYTES}" - echo "Annotating node objects with ${annotation_size_bytes} byte label" - label=$( (LC_CTYPE=C < /dev/urandom tr -dc 'a-zA-Z0-9' | fold -w "$annotation_size_bytes"; true) | head -n 1) - "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get nodes -o name \ - | xargs -P50 -r -I% "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" annotate --overwrite % label="$label" - echo "Annotating node objects completed" -} - - -detect-project &> /dev/null -create-kubemark-master - -if [ -f "${INTERNAL_KUBECONFIG}" ]; then - HOLLOWNODE_KUBECONFIG="${INTERNAL_KUBECONFIG}" -else - HOLLOWNODE_KUBECONFIG="${LOCAL_KUBECONFIG}" -fi - -MASTER_IP=$(grep server "${HOLLOWNODE_KUBECONFIG}" | awk -F "/" '{print $3}') - -start-hollow-nodes -resize-node-objects - -echo "" -echo "Master IP: ${MASTER_IP}" -echo "Kubeconfig for kubemark master is written in ${LOCAL_KUBECONFIG}" diff --git a/test/kubemark/stop-kubemark.sh b/test/kubemark/stop-kubemark.sh deleted file mode 100755 index e1bf6abdabd..00000000000 --- a/test/kubemark/stop-kubemark.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2015 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Script that destroys Kubemark cluster and deletes all master resources. - -KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../.. - -source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh" -source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh" -source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh" -source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh" - -if [[ -f "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/shutdown.sh" ]] ; then - source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/shutdown.sh" -fi - -source "${KUBE_ROOT}/cluster/kubemark/util.sh" - -KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh" -KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark" -RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources" - -detect-project &> /dev/null - -"${KUBECTL}" delete --timeout=5m -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" &> /dev/null || true -"${KUBECTL}" delete --timeout=5m -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark" &> /dev/null || true -"${KUBECTL}" delete --timeout=5m -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" &> /dev/null || true - -rm -rf "${RESOURCE_DIRECTORY}/addons" \ - "${RESOURCE_DIRECTORY}/kubeconfig.kubemark" \ - "${RESOURCE_DIRECTORY}/hollow-node.yaml" &> /dev/null || true - -delete-kubemark-master