delete kubemark

2026-11-06 05:17:08 +00:00 · 2026-10-06 00:30:23 +01:00
1 parent 7708623c1a
commit 8739d508bd
@@ -122,8 +122,6 @@ dependencies:
    refPaths:
    - path: test/e2e_node/image_list.go
      match: const defaultImage
-    - path: test/kubemark/resources/hollow-node_template.yaml
-      match: registry.k8s.io/node-problem-detector/node-problem-detector
    #- path: cluster/gce/windows/k8s-node-setup.psm1
    #  match: DEFAULT_NPD_VERSION

@@ -21,8 +21,6 @@ spec:
      capabilities:
        drop:
        - all
-    # When updating version also bump it in:
-    # - test/kubemark/resources/manifests/kube-addon-manager.yaml
    image: registry.k8s.io/addon-manager/kube-addon-manager:v9.1.8
    command:
    - /bin/bash
@@ -1,24 +0,0 @@
-# Copyright 2016 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The line below points to the latest go-runner image, a wrapper around
-# gcr.io/distroless/static which adds the go-runner command that we need
-# for redirecting log output.
-#
-# See https://console.cloud.google.com/gcr/images/k8s-staging-build-image/global/go-runner
-# for a list of available versions. This base image should be updated
-# periodically.
-FROM registry.k8s.io/build-image/go-runner:v2.3.1-go1.17.2-bullseye.0
-
-COPY kubemark /kubemark
@@ -1,36 +0,0 @@
-# Copyright 2016 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# build Kubemark image from currently built binaries containing both 'real' master and Hollow Node.
-# This makefile assumes that the kubemark binary is present in this directory.
-
-# Allow the caller to override this.  Beware make's precedence. This:
-#   REGISTRY=$VAR make
-# .. is not the same as:
-#   make REGISTRY=$VAR
-REGISTRY := $(if $(REGISTRY),$(REGISTRY),staging-k8s.gcr.io)
-IMAGE_TAG := $(if $(IMAGE_TAG),$(IMAGE_TAG),latest)
-
-all: gcloudpush
-
-build:
-	docker build --pull -t $(REGISTRY)/kubemark:$(IMAGE_TAG) .
-
-gcloudpush: build
-	docker push $(REGISTRY)/kubemark:$(IMAGE_TAG)
-
-push: build
-	docker push $(REGISTRY)/kubemark:$(IMAGE_TAG)
-
-.PHONY: all build gcloudpush push
@@ -1,12 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-reviewers:
-  - shyamjvs
-  - wojtek-t
-approvers:
-  - shyamjvs
-  - wojtek-t
-emeritus_approvers:
-  - gmarek
-labels:
-  - sig/scalability
@@ -1,14 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-reviewers:
-  - shyamjvs
-  - sig-scalability-reviewers
-  - wojtek-t
-approvers:
-  - shyamjvs
-  - sig-scalability-approvers
-  - wojtek-t
-emeritus_approvers:
-  - gmarek
-labels:
-  - sig/scalability
@@ -1,61 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A configuration for Kubemark cluster. It doesn't need to be kept in
-# sync with gce/config-default.sh (except the filename, because I'm reusing
-# gce/util.sh script which assumes config filename), but if some things that
-# are enabled by default should not run in hollow clusters, they should be disabled here.
-
-# shellcheck disable=SC2034 # Variables sourced in other scripts.
-
-source "${KUBE_ROOT}/cluster/gce/config-common.sh"
-
-CLEANUP_KUBEMARK_IMAGE=${CLEANUP_KUBEMARK_IMAGE:-true}
-TEST_CLUSTER_LOG_LEVEL="${TEST_CLUSTER_LOG_LEVEL:-"--v=4"}"
-
-# If you want to set up multiple kubemark clusters with different "names",
-# you should change this env per each start-kubemark.sh invocation.
-KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX:-"default"}"
-
-# NUM_NODES is used by start-kubemark.sh to determine a correct number of replicas.
-NUM_NODES=${KUBEMARK_NUM_NODES:-10}
-NUM_WINDOWS_NODES=${KUBEMARK_NUM_WINDOWS_NODES:-0}
-
-HOLLOW_KUBELET_TEST_LOG_LEVEL="${HOLLOW_KUBELET_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
-HOLLOW_PROXY_TEST_LOG_LEVEL="${HOLLOW_PROXY_TEST_LOG_LEVEL:-$TEST_CLUSTER_LOG_LEVEL}"
-
-# Hollow-node components' test arguments.
-HOLLOW_KUBELET_TEST_ARGS="${HOLLOW_KUBELET_TEST_ARGS:-} ${HOLLOW_KUBELET_TEST_LOG_LEVEL}"
-HOLLOW_PROXY_TEST_ARGS="${HOLLOW_PROXY_TEST_ARGS:-} ${HOLLOW_PROXY_TEST_LOG_LEVEL}"
-
-# Optional: Enable cluster autoscaler.
-ENABLE_KUBEMARK_CLUSTER_AUTOSCALER="${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-false}"
-# When using Cluster Autoscaler, always start with one hollow-node replica.
-# NUM_NODES should not be specified by the user. Instead we use
-# NUM_NODES=KUBEMARK_AUTOSCALER_MAX_NODES. This gives other cluster components
-# (e.g. kubemark master, Heapster) enough resources to handle maximum cluster size.
-if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER}" == "true" ]]; then
-  NUM_REPLICAS=1
-  if [[ -n "$NUM_NODES" ]]; then
-    echo "WARNING: Using Cluster Autoscaler, ignoring NUM_NODES parameter. Set KUBEMARK_AUTOSCALER_MAX_NODES to specify maximum size of the cluster."
-  fi
-fi
-
-#Optional: Enable kube dns.
-ENABLE_KUBEMARK_KUBE_DNS="${ENABLE_KUBEMARK_KUBE_DNS:-true}"
-KUBE_DNS_DOMAIN="${KUBE_DNS_DOMAIN:-cluster.local}"
-
-CLEANUP_KUBEMARK_IMAGE=false
@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2018 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Cloud information
-RANDGEN=$(dd if=/dev/urandom bs=64 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=16 count=1 2>/dev/null | sed 's/[A-Z]//g')
-# shellcheck disable=2034 # Variable sourced in other scripts.
-KUBE_NAMESPACE="kubemark_${RANDGEN}"
-KUBEMARK_IMAGE_TAG="${KUBEMARK_IMAGE_TAG:-2}"
-KUBEMARK_IMAGE_LOCATION="${KUBEMARK_IMAGE_LOCATION:-${KUBE_ROOT}/cluster/images/kubemark}"
-KUBEMARK_INIT_TAG="${KUBEMARK_INIT_TAG:-${PROJECT}:${KUBEMARK_IMAGE_TAG}}"
-CLUSTER_LOCATION="${CLUSTER_LOCATION:-wdc06}"
-REGISTRY_LOGIN_URL="${REGISTRY_LOGIN_URL:-https://api.ng.bluemix.net}"
-
-# User defined
-# number of real workers in spawnTester cluster
-NUM_NODES="${NUM_NODES:-2}"
-# spec of real workers in spawnTester cluster
-NODE_SIZE=${NODE_SIZE:-u2c.2x4}
-DESIRED_NODES="${DESIRED_NODES:-10}"
-# number of hollow nodes
-# TODO: once IKS supports `kubetest`, i.e. the base cluster provisioning implemented in `kubetest`
-# we can use NUM_NODES=${KUBEMARK_NUM_NODES:-10} to avoid usage of KUBEMARK_NUM_NODES
-# more context, see https://github.com/kubernetes/kubernetes/pull/76909#discussion_r277735942
-KUBEMARK_NUM_NODES="${KUBEMARK_NUM_NODES:-10}"
-ENABLE_KUBEMARK_CLUSTER_AUTOSCALER="${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-true}"
-ENABLE_KUBEMARK_KUBE_DNS="${ENABLE_KUBEMARK_KUBE_DNS:-false}"
-KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-"--v=2"}"
-KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-"--v=4"}"
-USE_REAL_PROXIER=${USE_REAL_PROXIER:-false}
-NODE_INSTANCE_PREFIX=${NODE_INSTANCE_PREFIX:-node}
-USE_EXISTING=${USE_EXISTING:-}
@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Configuration for landing a Kubemark cluster on a pre-existing Kubernetes
-# cluster.
-
-# Pre-existing provider expects a MASTER_IP.
-# If you need to specify a port that's not the default (443), add it to MASTER_IP.
-#
-# Example: Connect to the Master on the secure port 6443
-#          MASTER_IP=192.168.122.5:6443
-#
-MASTER_IP="${MASTER_IP:-}"
-
-# The container registry and project given to the kubemark container:
-#   $CONTAINER_REGISTRY/$PROJECT/kubemark
-#
-CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-}"
-PROJECT="${PROJECT:-}"
-
-NUM_NODES="${NUM_NODES:-1}"
-
-KUBELET_TEST_LOG_LEVEL="${KUBELET_TEST_LOG_LEVEL:-}"
-KUBEPROXY_TEST_LOG_LEVEL="${KUBEPROXY_TEST_LOG_LEVEL:-}"
-MASTER_NAME="${MASTER_NAME:-}"
@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2014 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
-source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
-source "${KUBE_ROOT}/cluster/${CLOUD_PROVIDER}/util.sh"
-source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"
@@ -44,7 +44,6 @@ readonly gcloud_supported_providers="gce gke"
 readonly master_logfiles="kube-apiserver.log kube-apiserver-audit.log kube-scheduler.log kube-controller-manager.log cloud-controller-manager.log etcd.log etcd-events.log glbc.log cluster-autoscaler.log kube-addon-manager.log konnectivity-server.log fluentd.log kubelet.cov"
 readonly node_logfiles="kube-proxy.log containers/konnectivity-agent-*.log fluentd.log kubelet.cov kube-network-policies.log"
 readonly node_systemd_services=""
-readonly hollow_node_logfiles="kubelet-hollow-node-*.log kubeproxy-hollow-node-*.log npd-hollow-node-*.log"
 readonly aws_logfiles="cloud-init-output.log"
 readonly gce_logfiles="startupscript.log"
 readonly kern_logfile="kern.log"
@@ -375,8 +374,6 @@ function dump_masters() {
  elif [[ ! "${master_ssh_supported_providers}" =~ ${KUBERNETES_PROVIDER} ]]; then
    echo "Master SSH not supported for ${KUBERNETES_PROVIDER}"
    return
-  elif [[ -n "${KUBEMARK_MASTER_NAME:-}" ]]; then
-    master_names=( "${KUBEMARK_MASTER_NAME}" )
  else
    if ! (detect-master); then
      echo 'Master not detected. Is the cluster up?'
@@ -444,9 +441,6 @@ function dump_nodes() {
  fi

  node_logfiles_all="${node_logfiles}"
-  if [[ "${ENABLE_HOLLOW_NODE_LOGS:-}" == "true" ]]; then
-    node_logfiles_all="${node_logfiles_all} ${hollow_node_logfiles}"
-  fi

  linux_nodes_selected_for_logs=()
  if [[ -n "${LOGDUMP_ONLY_N_RANDOM_NODES:-}" ]]; then
@@ -543,7 +537,6 @@ function dump_nodes_with_logexporter() {
  # Obtain parameters required by logexporter.
  local -r service_account_credentials="$(base64 "${GOOGLE_APPLICATION_CREDENTIALS}" | tr -d '\n')"
  local -r cloud_provider="${KUBERNETES_PROVIDER}"
-  local -r enable_hollow_node_logs="${ENABLE_HOLLOW_NODE_LOGS:-false}"
  local -r logexport_sleep_seconds="$(( 90 + NUM_NODES / 3 ))"
  if [[ -z "${ZONE_NODE_SELECTOR_DISABLED:-}" ]]; then
    local -r node_selector="${ZONE_NODE_SELECTOR_LABEL:-topology.kubernetes.io/zone}: ${ZONE}"
@@ -560,7 +553,6 @@ function dump_nodes_with_logexporter() {
  sed -i'' -e "s@{{.ServiceAccountCredentials}}@${service_account_credentials}@g" "${manifest_yaml}"
  sed -i'' -e "s@{{.CloudProvider}}@${cloud_provider}@g" "${manifest_yaml}"
  sed -i'' -e "s@{{.GCSPath}}@${gcs_artifacts_dir}@g" "${manifest_yaml}"
-  sed -i'' -e "s@{{.EnableHollowNodeLogs}}@${enable_hollow_node_logs}@g" "${manifest_yaml}"
  sed -i'' -e "s@{{.DumpSystemdJournal}}@${dump_systemd_journal}@g" "${manifest_yaml}"
  sed -i'' -e "s@{{.ExtraLogFiles}}@${extra_log_files}@g" "${manifest_yaml}"
  sed -i'' -e "s@{{.ExtraSystemdServices}}@${extra_systemd_services}@g" "${manifest_yaml}"
@@ -48,7 +48,6 @@ spec:
        - --cloud-provider={{.CloudProvider}}
        - --gcs-path={{.GCSPath}}
        - --gcloud-auth-file-path=/etc/service-account/service-account.json
-        - --enable-hollow-node-logs={{.EnableHollowNodeLogs}}
        - --dump-systemd-journal={{.DumpSystemdJournal}}
        - --extra-log-files={{.ExtraLogFiles}}
        - --extra-systemd-services={{.ExtraSystemdServices}}
@@ -1,4 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-labels:
-  - sig/scalability
@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A library of helper functions for landing kubemark containers on a
-# pre-existing Kubernetes master. See test/kubemark/pre-existing/README.md
-# for me details on using a pre-existing provider.
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
-
-source "${KUBE_ROOT}/cluster/common.sh"
-source "${KUBE_ROOT}/hack/lib/util.sh"
-
-function detect-project() {
-  if [[ -z "${MASTER_IP:-}" ]]; then
-    echo "Set 'MASTER_IP' to the instance assigned to be the Kubernetes master" 1>&2
-    exit 1
-  fi
-
-  if [[ -z "${PROJECT:-}" ]]; then
-    echo "Set 'PROJECT' to the name of the container project: $CONTAINER_REGISTRY/$PROJECT/kubemark" >&2
-    exit 1
-  fi
-
-  if [[ -z "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then
-    cluster_range=$(echo "${MASTER_IP}" | awk -F '.' '{printf("%d.%d.%d.0", $1, $2, $3)}')
-    SERVICE_CLUSTER_IP_RANGE="${SERVICE_CLUSTER_IP_RANGE:-$cluster_range/16}"
-  fi
-}
-
-function create-certs {
-  execute-cmd-on-pre-existing-master-with-retries 'sudo cat /etc/kubernetes/admin.conf' > /tmp/kubeconfig
-}
@@ -1,12 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-reviewers:
-  - shyamjvs
-  - sig-scalability-reviewers
-  - wojtek-t
-approvers:
-  - shyamjvs
-  - sig-scalability-approvers
-  - wojtek-t
-emeritus_approvers:
-  - gmarek
@@ -1,301 +0,0 @@
-/*
-Copyright 2023 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package app
-
-import (
-	"context"
-	goflag "flag"
-	"fmt"
-	"time"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
-	internalapi "k8s.io/cri-api/pkg/apis"
-	"k8s.io/klog/v2"
-
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/sets"
-	clientset "k8s.io/client-go/kubernetes"
-	restclient "k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/events"
-	cliflag "k8s.io/component-base/cli/flag"
-	_ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration
-	_ "k8s.io/component-base/metrics/prometheus/version"    // for version metric registration
-	"k8s.io/component-base/version"
-	"k8s.io/component-base/version/verflag"
-	remote "k8s.io/cri-client/pkg"
-	fakeremote "k8s.io/cri-client/pkg/fake"
-	"k8s.io/kubernetes/pkg/api/legacyscheme"
-	"k8s.io/kubernetes/pkg/cluster/ports"
-	cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
-	"k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap"
-	"k8s.io/kubernetes/pkg/kubelet/cm"
-	"k8s.io/kubernetes/pkg/kubemark"
-	kubemarkproxy "k8s.io/kubernetes/pkg/proxy/kubemark"
-	utilflag "k8s.io/kubernetes/pkg/util/flag"
-)
-
-type hollowNodeConfig struct {
-	KubeconfigPath          string
-	BootstrapKubeconfigPath string
-	CertDirectory           string
-	KubeletPort             int
-	KubeletReadOnlyPort     int
-	Morph                   string
-	NodeName                string
-	ServerPort              int
-	ContentType             string
-	QPS                     float32
-	Burst                   int
-	NodeLabels              map[string]string
-	RegisterWithTaints      []v1.Taint
-	MaxPods                 int
-	ExtendedResources       map[string]string
-	UseHostImageService     bool
-
-	// Deprecated config; remove these with the corresponding flags
-	UseRealProxier       bool
-	ProxierSyncPeriod    time.Duration
-	ProxierMinSyncPeriod time.Duration
-}
-
-const (
-	maxPods     = 110
-	podsPerCore = 0
-)
-
-// TODO(#45650): Refactor hollow-node into hollow-kubelet and hollow-proxy
-// and make the config driven.
-var knownMorphs = sets.NewString("kubelet", "proxy")
-
-func (c *hollowNodeConfig) addFlags(fs *pflag.FlagSet) {
-	fs.StringVar(&c.KubeconfigPath, "kubeconfig", "/kubeconfig/kubeconfig", "Path to kubeconfig file.")
-	fs.StringVar(&c.BootstrapKubeconfigPath, "bootstrap-kubeconfig", "", "Path to bootstrap kubeconfig file.")
-	fs.StringVar(&c.CertDirectory, "cert-dir", "/etc/srv/", "Path to cert directory for bootstraping.")
-	fs.IntVar(&c.KubeletPort, "kubelet-port", ports.KubeletPort, "Port on which HollowKubelet should be listening.")
-	fs.IntVar(&c.KubeletReadOnlyPort, "kubelet-read-only-port", ports.KubeletReadOnlyPort, "Read-only port on which Kubelet is listening.")
-	fs.StringVar(&c.NodeName, "name", "fake-node", "Name of this Hollow Node.")
-	fs.IntVar(&c.ServerPort, "api-server-port", 443, "Port on which API server is listening.")
-	fs.StringVar(&c.Morph, "morph", "", fmt.Sprintf("Specifies into which Hollow component this binary should morph. Allowed values: %v", knownMorphs.List()))
-	fs.StringVar(&c.ContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType of requests sent to apiserver.")
-	fs.Float32Var(&c.QPS, "kube-api-qps", 10, "QPS indicates the maximum QPS to the apiserver.")
-	fs.IntVar(&c.Burst, "kube-api-burst", 20, "Burst indicates maximum burst for throttle to the apiserver.")
-
-	bindableNodeLabels := cliflag.ConfigurationMap(c.NodeLabels)
-	fs.Var(&bindableNodeLabels, "node-labels", "Additional node labels")
-	fs.Var(utilflag.RegisterWithTaintsVar{Value: &c.RegisterWithTaints}, "register-with-taints", "Register the node with the given list of taints (comma separated \"<key>=<value>:<effect>\"). No-op if register-node is false.")
-	fs.IntVar(&c.MaxPods, "max-pods", maxPods, "Number of pods that can run on this Kubelet.")
-	bindableExtendedResources := cliflag.ConfigurationMap(c.ExtendedResources)
-	fs.Var(&bindableExtendedResources, "extended-resources", "Register the node with extended resources (comma separated \"<name>=<quantity>\")")
-	fs.BoolVar(&c.UseHostImageService, "use-host-image-service", true, "Set to true if the hollow-kubelet should use the host image service. If set to false the fake image service will be used")
-
-	fs.BoolVar(&c.UseRealProxier, "use-real-proxier", true, "Has no effect.")
-	_ = fs.MarkDeprecated("use-real-proxier", "This flag is deprecated and will be removed in a future release.")
-	fs.DurationVar(&c.ProxierSyncPeriod, "proxier-sync-period", 30*time.Second, "Has no effect.")
-	_ = fs.MarkDeprecated("proxier-sync-period", "This flag is deprecated and will be removed in a future release.")
-	fs.DurationVar(&c.ProxierMinSyncPeriod, "proxier-min-sync-period", 0, "Has no effect.")
-	_ = fs.MarkDeprecated("proxier-min-sync-period", "This flag is deprecated and will be removed in a future release.")
-}
-
-func (c *hollowNodeConfig) createClientConfigFromFile() (*restclient.Config, error) {
-	clientConfig, err := clientcmd.LoadFromFile(c.KubeconfigPath)
-	if err != nil {
-		return nil, fmt.Errorf("error while loading kubeconfig from file %v: %v", c.KubeconfigPath, err)
-	}
-	config, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig()
-	if err != nil {
-		return nil, fmt.Errorf("error while creating kubeconfig: %v", err)
-	}
-	config.ContentType = c.ContentType
-	config.QPS = c.QPS
-	config.Burst = c.Burst
-	return config, nil
-}
-
-func (c *hollowNodeConfig) bootstrapClientConfig() error {
-	if c.BootstrapKubeconfigPath != "" {
-		return bootstrap.LoadClientCert(context.TODO(), c.KubeconfigPath, c.BootstrapKubeconfigPath, c.CertDirectory, types.NodeName(c.NodeName))
-	}
-	return nil
-}
-
-func (c *hollowNodeConfig) createHollowKubeletOptions() *kubemark.HollowKubeletOptions {
-	return &kubemark.HollowKubeletOptions{
-		NodeName:            c.NodeName,
-		KubeletPort:         c.KubeletPort,
-		KubeletReadOnlyPort: c.KubeletReadOnlyPort,
-		MaxPods:             c.MaxPods,
-		PodsPerCore:         podsPerCore,
-		NodeLabels:          c.NodeLabels,
-		RegisterWithTaints:  c.RegisterWithTaints,
-	}
-}
-
-// NewHollowNodeCommand creates a *cobra.Command object with default parameters
-func NewHollowNodeCommand() *cobra.Command {
-	s := &hollowNodeConfig{
-		NodeLabels:        make(map[string]string),
-		ExtendedResources: make(map[string]string),
-	}
-
-	cmd := &cobra.Command{
-		Use:  "kubemark",
-		Long: "kubemark",
-		RunE: func(cmd *cobra.Command, args []string) error {
-			verflag.PrintAndExitIfRequested()
-			cliflag.PrintFlags(cmd.Flags())
-			return run(cmd.Context(), s)
-		},
-		Args: func(cmd *cobra.Command, args []string) error {
-			for _, arg := range args {
-				if len(arg) > 0 {
-					return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args)
-				}
-			}
-			return nil
-		},
-	}
-
-	fs := cmd.Flags()
-	fs.AddGoFlagSet(goflag.CommandLine) // for flags like --docker-only
-	s.addFlags(fs)
-
-	return cmd
-}
-
-func run(ctx context.Context, config *hollowNodeConfig) error {
-	// To help debugging, immediately log version and print flags.
-	klog.Infof("Version: %+v", version.Get())
-
-	if !knownMorphs.Has(config.Morph) {
-		return fmt.Errorf("Unknown morph: %v. allowed values: %v", config.Morph, knownMorphs.List())
-	}
-
-	// create a client to communicate with API server.
-	err := config.bootstrapClientConfig()
-	if err != nil {
-		return fmt.Errorf("Failed to bootstrap, error: %w. Exiting", err)
-	}
-	clientConfig, err := config.createClientConfigFromFile()
-	if err != nil {
-		return fmt.Errorf("Failed to create a ClientConfig, error: %w. Exiting", err)
-	}
-
-	if config.Morph == "kubelet" {
-		clientConfig.UserAgent = "hollow-kubelet"
-		client, err := clientset.NewForConfig(clientConfig)
-		if err != nil {
-			return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err)
-		}
-
-		f, c := kubemark.GetHollowKubeletConfig(config.createHollowKubeletOptions())
-
-		heartbeatClientConfig := *clientConfig
-		heartbeatClientConfig.Timeout = c.NodeStatusUpdateFrequency.Duration
-		// The timeout is the minimum of the lease duration and status update frequency
-		leaseTimeout := time.Duration(c.NodeLeaseDurationSeconds) * time.Second
-		if heartbeatClientConfig.Timeout > leaseTimeout {
-			heartbeatClientConfig.Timeout = leaseTimeout
-		}
-
-		heartbeatClientConfig.QPS = float32(-1)
-		heartbeatClient, err := clientset.NewForConfig(&heartbeatClientConfig)
-		if err != nil {
-			return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err)
-		}
-
-		cadvisorInterface := &cadvisortest.Fake{
-			NodeName: config.NodeName,
-		}
-
-		var containerManager cm.ContainerManager
-		if config.ExtendedResources != nil {
-			extendedResources := v1.ResourceList{}
-			for k, v := range config.ExtendedResources {
-				extendedResources[v1.ResourceName(k)] = resource.MustParse(v)
-			}
-
-			containerManager = cm.NewStubContainerManagerWithDevicePluginResource(extendedResources)
-		} else {
-			containerManager = cm.NewStubContainerManager()
-		}
-
-		endpoint, err := fakeremote.GenerateEndpoint()
-		if err != nil {
-			return fmt.Errorf("Failed to generate fake endpoint, error: %w", err)
-		}
-		fakeRemoteRuntime := fakeremote.NewFakeRemoteRuntime()
-		if err = fakeRemoteRuntime.Start(endpoint); err != nil {
-			return fmt.Errorf("Failed to start fake runtime, error: %w", err)
-		}
-		defer fakeRemoteRuntime.Stop()
-		runtimeService, err := remote.NewRemoteRuntimeServiceBuilder().
-			WithEndpoint(endpoint).
-			WithConnectionTimeout(15 * time.Second).
-			Build(ctx)
-		if err != nil {
-			return fmt.Errorf("Failed to init runtime service, error: %w", err)
-		}
-
-		var imageService internalapi.ImageManagerService = fakeRemoteRuntime.ImageService
-		if config.UseHostImageService {
-			imageService, err = remote.NewRemoteImageServiceBuilder().
-				WithEndpoint(c.ImageServiceEndpoint).
-				WithConnectionTimeout(15 * time.Second).
-				Build(ctx)
-			if err != nil {
-				return fmt.Errorf("Failed to init image service, error: %w", err)
-			}
-		}
-
-		hollowKubelet := kubemark.NewHollowKubelet(
-			f, c,
-			client,
-			heartbeatClient,
-			cadvisorInterface,
-			imageService,
-			runtimeService,
-			containerManager,
-		)
-		hollowKubelet.Run(ctx)
-	}
-
-	if config.Morph == "proxy" {
-		clientConfig.UserAgent = "hollow-proxy"
-
-		client, err := clientset.NewForConfig(clientConfig)
-		if err != nil {
-			return fmt.Errorf("Failed to create API Server client, error: %w", err)
-		}
-		eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()})
-		recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, "kube-proxy")
-
-		hollowProxy := kubemarkproxy.NewHollowProxy(
-			config.NodeName,
-			client,
-			client.CoreV1(),
-			eventBroadcaster,
-			recorder,
-		)
-		return hollowProxy.Run()
-	}
-
-	return nil
-}
@@ -1,96 +0,0 @@
-/*
-Copyright 2023 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package app
-
-import (
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"k8s.io/kubernetes/test/utils/ktesting"
-)
-
-const fakeKubeconfig = `
-apiVersion: v1
-kind: Config
-clusters:
- cluster:
-    insecure-skip-tls-verify: true
-    server: %s
-  name: default
-contexts:
- context:
-    cluster: default
-    user: default
-  name: default
-current-context: default
-users:
- name: default
-  user:
-    username: config
-`
-
-// TestHollowNode is a naive test that attempts to start hollow node and checks if it's not crashing.
-// Such test is sufficient to detect e.g. missing kubelet dependencies that are not added in
-// pkg/kubemark/hollow_kubelet.go.
-func TestHollowNode(t *testing.T) {
-	// temp dir
-	tmpDir, err := os.MkdirTemp("", "hollow-node")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tmpDir)
-
-	// https server
-	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		w.WriteHeader(200)
-		w.Write([]byte(`ok`))
-	}))
-	defer server.Close()
-
-	kubeconfigPath := filepath.Join(tmpDir, "config.kubeconfig")
-	if err := os.WriteFile(kubeconfigPath, []byte(fmt.Sprintf(fakeKubeconfig, server.URL)), os.FileMode(0600)); err != nil {
-		t.Fatal(err)
-	}
-
-	for morph := range knownMorphs {
-		t.Run(morph, func(t *testing.T) {
-			s := &hollowNodeConfig{
-				KubeconfigPath: kubeconfigPath,
-				Morph:          morph,
-			}
-			errCh := make(chan error)
-			go func() {
-				data, err := os.ReadFile(kubeconfigPath)
-				t.Logf("read %d, err=%v\n", len(data), err)
-				ctx := ktesting.Init(t)
-				errCh <- run(ctx, s)
-			}()
-
-			select {
-			case err := <-errCh:
-				t.Fatalf("Run finished unexpectedly with error: %v", err)
-			case <-time.After(3 * time.Second):
-				t.Logf("Morph %q hasn't crashed for 3s. Calling success.", morph)
-			}
-		})
-	}
-}
@@ -1,30 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"os"
-
-	"k8s.io/component-base/cli"
-	"k8s.io/kubernetes/cmd/kubemark/app"
-)
-
-func main() {
-	command := app.NewHollowNodeCommand()
-	code := cli.Run(command)
-	os.Exit(code)
-}
@@ -277,7 +277,6 @@ readonly KUBE_TEST_BINARIES=("${KUBE_TEST_TARGETS[@]##*/}")
 readonly KUBE_TEST_BINARIES_WIN=("${KUBE_TEST_BINARIES[@]/%/.exe}")
 readonly KUBE_TEST_PORTABLE=(
  test/e2e/testing-manifests
-  test/kubemark
  hack/e2e-internal
  hack/get-build.sh
  hack/ginkgo-e2e.sh
@@ -289,7 +288,6 @@ readonly KUBE_TEST_PORTABLE=(
 # These binaries will be distributed in the kubernetes-test tarball.
 kube::golang::server_test_targets() {
  local targets=(
-    cmd/kubemark
    ginkgo
  )

@@ -332,7 +330,6 @@ readonly KUBE_STATIC_BINARIES=(
  kubectl
  kubectl-convert
  kubelet
-  kubemark
  mounter
 )

@@ -4,7 +4,6 @@ rules:
      - ''
    forbiddenPrefixes:
      # prevent pkg/ from depending on cmd/
-      #   note: pkg/kubemark overrides this
      - k8s.io/kubernetes/cmd
      # use sigs.k8s.io/yaml instead
      - github.com/ghodss/yaml
@@ -1,5 +0,0 @@
-rules:
-  # override pkg/ import restriction on cmd/ for kubemark
-  - selectorRegexp: k8s[.]io/kubernetes/cmd
-    allowedPrefixes:
-      - k8s.io/kubernetes/cmd/kubelet/app
@@ -1,14 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-reviewers:
-  - shyamjvs
-  - sig-scalability-reviewers
-  - wojtek-t
-approvers:
-  - shyamjvs
-  - sig-scalability-approvers
-  - wojtek-t
-emeritus_approvers:
-  - gmarek
-labels:
-  - sig/scalability
@@ -1,401 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kubemark
-
-import (
-	"context"
-	"fmt"
-	"math/rand"
-	"sync"
-	"time"
-
-	apiv1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/fields"
-	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/client-go/informers"
-	informersv1 "k8s.io/client-go/informers/core/v1"
-	kubeclient "k8s.io/client-go/kubernetes"
-	listersv1 "k8s.io/client-go/listers/core/v1"
-	"k8s.io/client-go/tools/cache"
-
-	"k8s.io/klog/v2"
-)
-
-const (
-	namespaceKubemark = "kubemark"
-	nodeGroupLabel    = "autoscaling.k8s.io/nodegroup"
-	numRetries        = 3
-)
-
-// KubemarkController is a simplified version of cloud provider for kubemark. It allows
-// to add and delete nodes from a kubemark cluster and introduces nodegroups
-// by applying labels to the kubemark's hollow-nodes.
-type KubemarkController struct {
-	nodeTemplate           *apiv1.ReplicationController
-	externalCluster        externalCluster
-	kubemarkCluster        kubemarkCluster
-	rand                   *rand.Rand
-	createNodeQueue        chan string
-	nodeGroupQueueSize     map[string]int
-	nodeGroupQueueSizeLock sync.Mutex
-}
-
-// externalCluster is used to communicate with the external cluster that hosts
-// kubemark, in order to be able to list, create and delete hollow nodes
-// by manipulating the replication controllers.
-type externalCluster struct {
-	rcLister  listersv1.ReplicationControllerLister
-	rcSynced  cache.InformerSynced
-	podLister listersv1.PodLister
-	podSynced cache.InformerSynced
-	client    kubeclient.Interface
-}
-
-// kubemarkCluster is used to delete nodes from kubemark cluster once their
-// respective replication controllers have been deleted and the nodes have
-// become unready. This is to cover for the fact that there is no proper cloud
-// provider for kubemark that would care for deleting the nodes.
-type kubemarkCluster struct {
-	client            kubeclient.Interface
-	nodeLister        listersv1.NodeLister
-	nodeSynced        cache.InformerSynced
-	nodesToDelete     map[string]bool
-	nodesToDeleteLock sync.Mutex
-}
-
-// NewKubemarkController creates KubemarkController using the provided clients to talk to external
-// and kubemark clusters.
-func NewKubemarkController(externalClient kubeclient.Interface, externalInformerFactory informers.SharedInformerFactory,
-	kubemarkClient kubeclient.Interface, kubemarkNodeInformer informersv1.NodeInformer) (*KubemarkController, error) {
-	rcInformer := externalInformerFactory.InformerFor(&apiv1.ReplicationController{}, newReplicationControllerInformer)
-	podInformer := externalInformerFactory.InformerFor(&apiv1.Pod{}, newPodInformer)
-	controller := &KubemarkController{
-		externalCluster: externalCluster{
-			rcLister:  listersv1.NewReplicationControllerLister(rcInformer.GetIndexer()),
-			rcSynced:  rcInformer.HasSynced,
-			podLister: listersv1.NewPodLister(podInformer.GetIndexer()),
-			podSynced: podInformer.HasSynced,
-			client:    externalClient,
-		},
-		kubemarkCluster: kubemarkCluster{
-			nodeLister:        kubemarkNodeInformer.Lister(),
-			nodeSynced:        kubemarkNodeInformer.Informer().HasSynced,
-			client:            kubemarkClient,
-			nodesToDelete:     make(map[string]bool),
-			nodesToDeleteLock: sync.Mutex{},
-		},
-		rand:                   rand.New(rand.NewSource(time.Now().UnixNano())),
-		createNodeQueue:        make(chan string, 1000),
-		nodeGroupQueueSize:     make(map[string]int),
-		nodeGroupQueueSizeLock: sync.Mutex{},
-	}
-
-	kubemarkNodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
-		UpdateFunc: controller.kubemarkCluster.removeUnneededNodes,
-	})
-
-	return controller, nil
-}
-
-// WaitForCacheSync waits until all caches in the controller are populated.
-func (kubemarkController *KubemarkController) WaitForCacheSync(stopCh chan struct{}) bool {
-	return cache.WaitForNamedCacheSync("kubemark", stopCh,
-		kubemarkController.externalCluster.rcSynced,
-		kubemarkController.externalCluster.podSynced,
-		kubemarkController.kubemarkCluster.nodeSynced)
-}
-
-// Run populates the node template needed for creation of kubemark nodes and
-// starts the worker routine for creating new nodes.
-func (kubemarkController *KubemarkController) Run(stopCh chan struct{}) {
-	nodeTemplate, err := kubemarkController.getNodeTemplate()
-	if err != nil {
-		klog.Fatalf("failed to get node template: %s", err)
-	}
-	kubemarkController.nodeTemplate = nodeTemplate
-
-	go kubemarkController.runNodeCreation(stopCh)
-	<-stopCh
-}
-
-// GetNodeNamesForNodeGroup returns list of the nodes in the node group.
-func (kubemarkController *KubemarkController) GetNodeNamesForNodeGroup(nodeGroup string) ([]string, error) {
-	selector := labels.SelectorFromSet(labels.Set{nodeGroupLabel: nodeGroup})
-	pods, err := kubemarkController.externalCluster.podLister.List(selector)
-	if err != nil {
-		return nil, err
-	}
-	result := make([]string, 0, len(pods))
-	for _, pod := range pods {
-		result = append(result, pod.ObjectMeta.Name)
-	}
-	return result, nil
-}
-
-// GetNodeGroupSize returns the current size for the node group as observed.
-func (kubemarkController *KubemarkController) GetNodeGroupSize(nodeGroup string) (int, error) {
-	selector := labels.SelectorFromSet(labels.Set(map[string]string{nodeGroupLabel: nodeGroup}))
-	nodes, err := kubemarkController.externalCluster.rcLister.List(selector)
-	if err != nil {
-		return 0, err
-	}
-	return len(nodes), nil
-}
-
-// GetNodeGroupTargetSize returns the size of the node group as a sum of current
-// observed size and number of upcoming nodes.
-func (kubemarkController *KubemarkController) GetNodeGroupTargetSize(nodeGroup string) (int, error) {
-	kubemarkController.nodeGroupQueueSizeLock.Lock()
-	defer kubemarkController.nodeGroupQueueSizeLock.Unlock()
-	realSize, err := kubemarkController.GetNodeGroupSize(nodeGroup)
-	if err != nil {
-		return realSize, err
-	}
-	return realSize + kubemarkController.nodeGroupQueueSize[nodeGroup], nil
-}
-
-// SetNodeGroupSize changes the size of node group by adding or removing nodes.
-func (kubemarkController *KubemarkController) SetNodeGroupSize(nodeGroup string, size int) error {
-	currSize, err := kubemarkController.GetNodeGroupTargetSize(nodeGroup)
-	if err != nil {
-		return err
-	}
-	switch delta := size - currSize; {
-	case delta < 0:
-		absDelta := -delta
-		nodes, err := kubemarkController.GetNodeNamesForNodeGroup(nodeGroup)
-		if err != nil {
-			return err
-		}
-		if len(nodes) < absDelta {
-			return fmt.Errorf("can't remove %d nodes from %s nodegroup, not enough nodes: %d", absDelta, nodeGroup, len(nodes))
-		}
-		for i, node := range nodes {
-			if i == absDelta {
-				return nil
-			}
-			if err := kubemarkController.RemoveNodeFromNodeGroup(nodeGroup, node); err != nil {
-				return err
-			}
-		}
-	case delta > 0:
-		kubemarkController.nodeGroupQueueSizeLock.Lock()
-		kubemarkController.nodeGroupQueueSize[nodeGroup] += delta
-		kubemarkController.nodeGroupQueueSizeLock.Unlock()
-		for i := 0; i < delta; i++ {
-			kubemarkController.createNodeQueue <- nodeGroup
-		}
-	}
-
-	return nil
-}
-
-// GetNodeGroupForNode returns the name of the node group to which the node
-// belongs.
-func (kubemarkController *KubemarkController) GetNodeGroupForNode(node string) (string, error) {
-	pod := kubemarkController.getPodByName(node)
-	if pod == nil {
-		return "", fmt.Errorf("node %s does not exist", node)
-	}
-	nodeGroup, ok := pod.ObjectMeta.Labels[nodeGroupLabel]
-	if ok {
-		return nodeGroup, nil
-	}
-	return "", fmt.Errorf("can't find nodegroup for node %s due to missing label %s", node, nodeGroupLabel)
-}
-
-func (kubemarkController *KubemarkController) addNodeToNodeGroup(nodeGroup string) error {
-	node := kubemarkController.nodeTemplate.DeepCopy()
-	node.Name = fmt.Sprintf("%s-%d", nodeGroup, kubemarkController.rand.Int63())
-	node.Labels = map[string]string{nodeGroupLabel: nodeGroup, "name": node.Name}
-	node.Spec.Template.Labels = node.Labels
-
-	var err error
-	for i := 0; i < numRetries; i++ {
-		_, err = kubemarkController.externalCluster.client.CoreV1().ReplicationControllers(node.Namespace).Create(context.TODO(), node, metav1.CreateOptions{})
-		if err == nil {
-			return nil
-		}
-	}
-
-	return err
-}
-
-func (kubemarkController *KubemarkController) RemoveNodeFromNodeGroup(nodeGroup string, node string) error {
-	pod := kubemarkController.getPodByName(node)
-	if pod == nil {
-		klog.Warningf("Can't delete node %s from nodegroup %s. Node does not exist.", node, nodeGroup)
-		return nil
-	}
-	if pod.ObjectMeta.Labels[nodeGroupLabel] != nodeGroup {
-		return fmt.Errorf("can't delete node %s from nodegroup %s. Node is not in nodegroup", node, nodeGroup)
-	}
-	policy := metav1.DeletePropagationForeground
-	var err error
-	for i := 0; i < numRetries; i++ {
-		err = kubemarkController.externalCluster.client.CoreV1().ReplicationControllers(namespaceKubemark).Delete(context.TODO(), pod.ObjectMeta.Labels["name"],
-			metav1.DeleteOptions{PropagationPolicy: &policy})
-		if err == nil {
-			klog.Infof("marking node %s for deletion", node)
-			// Mark node for deletion from kubemark cluster.
-			// Once it becomes unready after replication controller
-			// deletion has been noticed, we will delete it explicitly.
-			// This is to cover for the fact that kubemark does not
-			// take care of this itself.
-			kubemarkController.kubemarkCluster.markNodeForDeletion(node)
-			return nil
-		}
-	}
-	return fmt.Errorf("Failed to delete node %s: %v", node, err)
-}
-
-func (kubemarkController *KubemarkController) getReplicationControllerByName(name string) *apiv1.ReplicationController {
-	rcs, err := kubemarkController.externalCluster.rcLister.List(labels.Everything())
-	if err != nil {
-		return nil
-	}
-	for _, rc := range rcs {
-		if rc.ObjectMeta.Name == name {
-			return rc
-		}
-	}
-	return nil
-}
-
-func (kubemarkController *KubemarkController) getPodByName(name string) *apiv1.Pod {
-	pods, err := kubemarkController.externalCluster.podLister.List(labels.Everything())
-	if err != nil {
-		return nil
-	}
-	for _, pod := range pods {
-		if pod.ObjectMeta.Name == name {
-			return pod
-		}
-	}
-	return nil
-}
-
-func (kubemarkController *KubemarkController) getNodeNameForPod(podName string) (string, error) {
-	pods, err := kubemarkController.externalCluster.podLister.List(labels.Everything())
-	if err != nil {
-		return "", err
-	}
-	for _, pod := range pods {
-		if pod.ObjectMeta.Name == podName {
-			return pod.Labels["name"], nil
-		}
-	}
-	return "", fmt.Errorf("pod %s not found", podName)
-}
-
-// getNodeTemplate returns the template for hollow node replication controllers
-// by looking for an existing hollow node specification. This requires at least
-// one kubemark node to be present on startup.
-func (kubemarkController *KubemarkController) getNodeTemplate() (*apiv1.ReplicationController, error) {
-	podName, err := kubemarkController.kubemarkCluster.getHollowNodeName()
-	if err != nil {
-		return nil, err
-	}
-	hollowNodeName, err := kubemarkController.getNodeNameForPod(podName)
-	if err != nil {
-		return nil, err
-	}
-	if hollowNode := kubemarkController.getReplicationControllerByName(hollowNodeName); hollowNode != nil {
-		nodeTemplate := &apiv1.ReplicationController{
-			Spec: apiv1.ReplicationControllerSpec{
-				Template: hollowNode.Spec.Template,
-			},
-		}
-
-		nodeTemplate.Spec.Selector = nil
-		nodeTemplate.Namespace = namespaceKubemark
-		one := int32(1)
-		nodeTemplate.Spec.Replicas = &one
-
-		return nodeTemplate, nil
-	}
-	return nil, fmt.Errorf("can't get hollow node template")
-}
-
-func (kubemarkController *KubemarkController) runNodeCreation(stop <-chan struct{}) {
-	for {
-		select {
-		case nodeGroup := <-kubemarkController.createNodeQueue:
-			kubemarkController.nodeGroupQueueSizeLock.Lock()
-			err := kubemarkController.addNodeToNodeGroup(nodeGroup)
-			if err != nil {
-				klog.Errorf("failed to add node to node group %s: %v", nodeGroup, err)
-			} else {
-				kubemarkController.nodeGroupQueueSize[nodeGroup]--
-			}
-			kubemarkController.nodeGroupQueueSizeLock.Unlock()
-		case <-stop:
-			return
-		}
-	}
-}
-
-func (kubemarkCluster *kubemarkCluster) getHollowNodeName() (string, error) {
-	selector, _ := labels.Parse(nodeGroupLabel)
-	nodes, err := kubemarkCluster.nodeLister.List(selector)
-	if err != nil {
-		return "", err
-	}
-	for _, node := range nodes {
-		return node.Name, nil
-	}
-	return "", fmt.Errorf("did not find any hollow nodes in the cluster")
-}
-
-func (kubemarkCluster *kubemarkCluster) removeUnneededNodes(oldObj interface{}, newObj interface{}) {
-	node, ok := newObj.(*apiv1.Node)
-	if !ok {
-		return
-	}
-	for _, condition := range node.Status.Conditions {
-		// Delete node if it is in unready state, and it has been
-		// explicitly marked for deletion.
-		if condition.Type == apiv1.NodeReady && condition.Status != apiv1.ConditionTrue {
-			kubemarkCluster.nodesToDeleteLock.Lock()
-			defer kubemarkCluster.nodesToDeleteLock.Unlock()
-			if kubemarkCluster.nodesToDelete[node.Name] {
-				kubemarkCluster.nodesToDelete[node.Name] = false
-				if err := kubemarkCluster.client.CoreV1().Nodes().Delete(context.TODO(), node.Name, metav1.DeleteOptions{}); err != nil {
-					klog.Errorf("failed to delete node %s from kubemark cluster, err: %v", node.Name, err)
-				}
-			}
-			return
-		}
-	}
-}
-
-func (kubemarkCluster *kubemarkCluster) markNodeForDeletion(name string) {
-	kubemarkCluster.nodesToDeleteLock.Lock()
-	defer kubemarkCluster.nodesToDeleteLock.Unlock()
-	kubemarkCluster.nodesToDelete[name] = true
-}
-
-func newReplicationControllerInformer(kubeClient kubeclient.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
-	rcListWatch := cache.NewListWatchFromClient(kubeClient.CoreV1().RESTClient(), "replicationcontrollers", namespaceKubemark, fields.Everything())
-	return cache.NewSharedIndexInformer(rcListWatch, &apiv1.ReplicationController{}, resyncPeriod, nil)
-}
-
-func newPodInformer(kubeClient kubeclient.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
-	podListWatch := cache.NewListWatchFromClient(kubeClient.CoreV1().RESTClient(), "pods", namespaceKubemark, fields.Everything())
-	return cache.NewSharedIndexInformer(podListWatch, &apiv1.Pod{}, resyncPeriod, nil)
-}
@@ -1,213 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kubemark
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	noopoteltrace "go.opentelemetry.io/otel/trace/noop"
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/klog/v2"
-	"k8s.io/mount-utils"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/record"
-	internalapi "k8s.io/cri-api/pkg/apis"
-	kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
-	"k8s.io/kubernetes/cmd/kubelet/app/options"
-	"k8s.io/kubernetes/pkg/kubelet"
-	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
-	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
-	"k8s.io/kubernetes/pkg/kubelet/cm"
-	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
-	probetest "k8s.io/kubernetes/pkg/kubelet/prober/testing"
-	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
-	kubeletutil "k8s.io/kubernetes/pkg/kubelet/util"
-	"k8s.io/kubernetes/pkg/util/oom"
-	"k8s.io/kubernetes/pkg/volume"
-	"k8s.io/kubernetes/pkg/volume/configmap"
-	"k8s.io/kubernetes/pkg/volume/csi"
-	"k8s.io/kubernetes/pkg/volume/downwardapi"
-	"k8s.io/kubernetes/pkg/volume/emptydir"
-	"k8s.io/kubernetes/pkg/volume/fc"
-	"k8s.io/kubernetes/pkg/volume/git_repo"
-	"k8s.io/kubernetes/pkg/volume/hostpath"
-	"k8s.io/kubernetes/pkg/volume/iscsi"
-	"k8s.io/kubernetes/pkg/volume/local"
-	"k8s.io/kubernetes/pkg/volume/nfs"
-	"k8s.io/kubernetes/pkg/volume/projected"
-	"k8s.io/kubernetes/pkg/volume/secret"
-	"k8s.io/kubernetes/pkg/volume/util/hostutil"
-	"k8s.io/kubernetes/pkg/volume/util/subpath"
-	"k8s.io/kubernetes/test/utils"
-)
-
-type HollowKubelet struct {
-	KubeletFlags         *options.KubeletFlags
-	KubeletConfiguration *kubeletconfig.KubeletConfiguration
-	KubeletDeps          *kubelet.Dependencies
-}
-
-func volumePlugins() []volume.VolumePlugin {
-	allPlugins := []volume.VolumePlugin{}
-	allPlugins = append(allPlugins, emptydir.ProbeVolumePlugins()...)
-	allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...)
-	allPlugins = append(allPlugins, hostpath.FakeProbeVolumePlugins(volume.VolumeConfig{})...)
-	allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(volume.VolumeConfig{})...)
-	allPlugins = append(allPlugins, secret.ProbeVolumePlugins()...)
-	allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...)
-	allPlugins = append(allPlugins, downwardapi.ProbeVolumePlugins()...)
-	allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
-	allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...)
-	allPlugins = append(allPlugins, projected.ProbeVolumePlugins()...)
-	allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
-	allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
-	return allPlugins
-}
-
-func NewHollowKubelet(
-	flags *options.KubeletFlags,
-	config *kubeletconfig.KubeletConfiguration,
-	client *clientset.Clientset,
-	heartbeatClient *clientset.Clientset,
-	cadvisorInterface cadvisor.Interface,
-	imageService internalapi.ImageManagerService,
-	runtimeService internalapi.RuntimeService,
-	containerManager cm.ContainerManager) *HollowKubelet {
-	// Kubemark dependency construction does not have a higher-level logger.
-	nodeStartupLatencyTracker := kubeletutil.NewNodeStartupLatencyTracker(klog.TODO())
-
-	d := &kubelet.Dependencies{
-		KubeClient:                client,
-		HeartbeatClient:           heartbeatClient,
-		ProbeManager:              probetest.FakeManager{},
-		RemoteRuntimeService:      runtimeService,
-		RemoteImageService:        imageService,
-		CAdvisorInterface:         cadvisorInterface,
-		OSInterface:               &containertest.FakeOS{},
-		ContainerManager:          containerManager,
-		VolumePlugins:             volumePlugins(),
-		TLSOptions:                nil,
-		OOMAdjuster:               oom.NewFakeOOMAdjuster(),
-		Mounter:                   &mount.FakeMounter{},
-		Subpather:                 &subpath.FakeSubpath{},
-		HostUtil:                  hostutil.NewFakeHostUtil(nil),
-		PodStartupLatencyTracker:  kubeletutil.NewPodStartupLatencyTracker(),
-		NodeStartupLatencyTracker: nodeStartupLatencyTracker,
-		TracerProvider:            noopoteltrace.NewTracerProvider(),
-		Recorder:                  &record.FakeRecorder{}, // With real recorder we attempt to read /dev/kmsg.
-	}
-
-	return &HollowKubelet{
-		KubeletFlags:         flags,
-		KubeletConfiguration: config,
-		KubeletDeps:          d,
-	}
-}
-
-// Starts this HollowKubelet and blocks.
-func (hk *HollowKubelet) Run(ctx context.Context) {
-	if err := kubeletapp.RunKubelet(ctx, &options.KubeletServer{
-		KubeletFlags:         *hk.KubeletFlags,
-		KubeletConfiguration: *hk.KubeletConfiguration,
-	}, hk.KubeletDeps); err != nil {
-		klog.Fatalf("Failed to run HollowKubelet: %v. Exiting.", err)
-	}
-	select {}
-}
-
-// HollowKubeletOptions contains settable parameters for hollow kubelet.
-type HollowKubeletOptions struct {
-	NodeName            string
-	KubeletPort         int
-	KubeletReadOnlyPort int
-	MaxPods             int
-	PodsPerCore         int
-	NodeLabels          map[string]string
-	RegisterWithTaints  []v1.Taint
-}
-
-// Builds a KubeletConfiguration for the HollowKubelet, ensuring that the
-// usual defaults are applied for fields we do not override.
-func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, *kubeletconfig.KubeletConfiguration) {
-	testRootDir := utils.MakeTempDirOrDie("hollow-kubelet.", "")
-	podFilePath := utils.MakeTempDirOrDie("static-pods", testRootDir)
-	podLogsPath := utils.MakeTempDirOrDie("pod-logs", testRootDir)
-	klog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)
-
-	// Flags struct
-	f := options.NewKubeletFlags()
-	f.RootDirectory = testRootDir
-	f.HostnameOverride = opt.NodeName
-	f.MinimumGCAge = metav1.Duration{Duration: 1 * time.Minute}
-	f.MaxContainerCount = 100
-	f.MaxPerPodContainerCount = 2
-	f.NodeLabels = opt.NodeLabels
-
-	// Config struct
-	c, err := options.NewKubeletConfiguration()
-	if err != nil {
-		panic(err)
-	}
-
-	c.ImageServiceEndpoint = "unix:///run/containerd/containerd.sock"
-	c.StaticPodURL = ""
-	c.EnableServer = true
-	c.Address = "0.0.0.0" /* bind address */
-	c.Port = int32(opt.KubeletPort)
-	c.ReadOnlyPort = int32(opt.KubeletReadOnlyPort)
-	c.StaticPodPath = podFilePath
-	c.FileCheckFrequency.Duration = 20 * time.Second
-	c.HTTPCheckFrequency.Duration = 20 * time.Second
-	c.NodeStatusUpdateFrequency.Duration = 10 * time.Second
-	c.NodeStatusReportFrequency.Duration = 5 * time.Minute
-	c.SyncFrequency.Duration = 10 * time.Second
-	c.EvictionPressureTransitionPeriod.Duration = 5 * time.Minute
-	c.MaxPods = int32(opt.MaxPods)
-	c.PodsPerCore = int32(opt.PodsPerCore)
-	c.ClusterDNS = []string{}
-	c.ImageGCHighThresholdPercent = 90
-	c.ImageGCLowThresholdPercent = 80
-	c.ProviderID = fmt.Sprintf("kubemark://%v", opt.NodeName)
-	c.VolumeStatsAggPeriod.Duration = time.Minute
-	c.CgroupRoot = ""
-	c.CPUCFSQuota = true
-	c.EnableControllerAttachDetach = false
-	c.EnableDebuggingHandlers = true
-	c.CgroupsPerQOS = false
-	// hairpin-veth is used to allow hairpin packets. Note that this deviates from
-	// what the "real" kubelet currently does, because there's no way to
-	// set promiscuous mode on docker0.
-	c.HairpinMode = kubeletconfig.HairpinVeth
-	c.MaxOpenFiles = 1024
-	c.RegistryBurst = 10
-	c.RegistryPullQPS = 5.0
-	c.ResolverConfig = kubetypes.ResolvConfDefault
-	c.KubeletCgroups = "/kubelet"
-	c.SerializeImagePulls = true
-	c.SystemCgroups = ""
-	c.ProtectKernelDefaults = false
-	c.RegisterWithTaints = opt.RegisterWithTaints
-	c.RegisterNode = true
-	c.LocalStorageCapacityIsolation = true
-	c.PodLogsDir = podLogsPath
-
-	return f, c
-}
@@ -1,6 +0,0 @@
-rules:
-  # override pkg/ import restriction on cmd/ for kubemark
-  - selectorRegexp: k8s[.]io/kubernetes/cmd
-    allowedPrefixes:
-      - k8s.io/kubernetes/cmd/kube-proxy/app
-
@@ -1,14 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-reviewers:
-  - shyamjvs
-  - sig-scalability-reviewers
-  - wojtek-t
-approvers:
-  - shyamjvs
-  - sig-scalability-approvers
-  - wojtek-t
-emeritus_approvers:
-  - gmarek
-labels:
-  - sig/scalability
@@ -1,93 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kubemark
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	v1 "k8s.io/api/core/v1"
-	discoveryv1 "k8s.io/api/discovery/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	clientset "k8s.io/client-go/kubernetes"
-	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
-	"k8s.io/client-go/tools/events"
-	proxyapp "k8s.io/kubernetes/cmd/kube-proxy/app"
-	kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
-	"k8s.io/utils/ptr"
-)
-
-type HollowProxy struct {
-	ProxyServer *proxyapp.ProxyServer
-}
-
-type FakeProxier struct{}
-
-func (*FakeProxier) Sync() {}
-func (*FakeProxier) SyncLoop() {
-	select {}
-}
-func (*FakeProxier) OnServiceAdd(service *v1.Service)                                 {}
-func (*FakeProxier) OnServiceUpdate(oldService, service *v1.Service)                  {}
-func (*FakeProxier) OnServiceDelete(service *v1.Service)                              {}
-func (*FakeProxier) OnServiceSynced()                                                 {}
-func (*FakeProxier) OnEndpointSliceAdd(slice *discoveryv1.EndpointSlice)              {}
-func (*FakeProxier) OnEndpointSliceUpdate(oldSlice, slice *discoveryv1.EndpointSlice) {}
-func (*FakeProxier) OnEndpointSliceDelete(slice *discoveryv1.EndpointSlice)           {}
-func (*FakeProxier) OnEndpointSlicesSynced()                                          {}
-func (*FakeProxier) OnServiceCIDRsChanged(_ []string)                                 {}
-func (*FakeProxier) OnTopologyChange(_ map[string]string)                             {}
-
-func NewHollowProxy(
-	nodeName string,
-	client clientset.Interface,
-	eventClient v1core.EventsGetter,
-	broadcaster events.EventBroadcaster,
-	recorder events.EventRecorder,
-) *HollowProxy {
-	return &HollowProxy{
-		ProxyServer: &proxyapp.ProxyServer{
-			Config: &kubeproxyconfig.KubeProxyConfiguration{
-				Mode:             kubeproxyconfig.ProxyMode("fake"),
-				ConfigSyncPeriod: metav1.Duration{Duration: 30 * time.Second},
-				Linux: kubeproxyconfig.KubeProxyLinuxConfiguration{
-					OOMScoreAdj: ptr.To[int32](0),
-				},
-			},
-
-			Client:      client,
-			Proxier:     &FakeProxier{},
-			Broadcaster: broadcaster,
-			Recorder:    recorder,
-			NodeRef: &v1.ObjectReference{
-				APIVersion: "v1",
-				Kind:       "Node",
-				Name:       nodeName,
-				Namespace:  "",
-			},
-		},
-	}
-}
-
-func (hp *HollowProxy) Run() error {
-
-	if err := hp.ProxyServer.Run(context.TODO()); err != nil {
-		return fmt.Errorf("Error while running proxy: %w", err)
-	}
-	return nil
-}
@@ -261,7 +261,7 @@ func verifyRemainingObjects(ctx context.Context, f *framework.Framework, objects
 func gatherMetrics(ctx context.Context, f *framework.Framework) {
 	ginkgo.By("Gathering metrics")
 	var summary framework.TestDataSummary
-	grabber, err := e2emetrics.NewMetricsGrabber(ctx, f.ClientSet, f.KubemarkExternalClusterClientSet, f.ClientConfig(), false, false, true, false, false, false)
+	grabber, err := e2emetrics.NewMetricsGrabber(ctx, f.ClientSet, f.ClientConfig(), false, false, true, false, false, false)
 	if err != nil {
 		framework.Logf("Failed to create MetricsGrabber. Skipping metrics gathering.")
 	} else {
@@ -69,7 +69,6 @@ func init() {
 					}

 					gatherer, err := e2edebug.NewResourceUsageGatherer(ctx, f.ClientSet, e2edebug.ResourceGathererOptions{
-						InKubemark:                  framework.ProviderIs("kubemark"),
 						Nodes:                       nodeMode,
 						ResourceDataGatheringPeriod: 60 * time.Second,
 						ProbeDuration:               15 * time.Second,
@@ -17,7 +17,6 @@ limitations under the License.
 package debug

 import (
-	"bufio"
 	"bytes"
 	"context"
 	"encoding/json"
@@ -40,7 +39,6 @@ import (
 	kubeletstatsv1alpha1 "k8s.io/kubelet/pkg/apis/stats/v1alpha1"

 	"k8s.io/kubernetes/test/e2e/framework"
-	e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
 	"k8s.io/utils/ptr"
 )

@@ -177,7 +175,6 @@ type resourceGatherWorker struct {
 	stopCh                      chan struct{}
 	dataSeries                  []ResourceUsagePerContainer
 	finished                    bool
-	inKubemark                  bool
 	resourceDataGatheringPeriod time.Duration
 	probeDuration               time.Duration
 	printVerboseLogs            bool
@@ -185,29 +182,15 @@ type resourceGatherWorker struct {

 func (w *resourceGatherWorker) singleProbe(ctx context.Context) {
 	data := make(ResourceUsagePerContainer)
-	if w.inKubemark {
-		kubemarkData := getKubemarkMasterComponentsResourceUsage(ctx)
-		if kubemarkData == nil {
-			return
-		}
-		for k, v := range kubemarkData {
-			data[k] = &ContainerResourceUsage{
-				Name:                    v.Name,
-				MemoryWorkingSetInBytes: v.MemoryWorkingSetInBytes,
-				CPUUsageInCores:         v.CPUUsageInCores,
-			}
-		}
-	} else {
-		nodeUsage, err := getOneTimeResourceUsageOnNode(w.c, w.nodeName, w.probeDuration, func() []string { return w.containerIDs })
-		if err != nil {
-			framework.Logf("Error while reading data from %v: %v", w.nodeName, err)
-			return
-		}
-		for k, v := range nodeUsage {
-			data[k] = v
-			if w.printVerboseLogs {
-				framework.Logf("Get container %v usage on node %v. CPUUsageInCores: %v, MemoryUsageInBytes: %v, MemoryWorkingSetInBytes: %v", k, w.nodeName, v.CPUUsageInCores, v.MemoryUsageInBytes, v.MemoryWorkingSetInBytes)
-			}
+	nodeUsage, err := getOneTimeResourceUsageOnNode(w.c, w.nodeName, w.probeDuration, func() []string { return w.containerIDs })
+	if err != nil {
+		framework.Logf("Error while reading data from %v: %v", w.nodeName, err)
+		return
+	}
+	for k, v := range nodeUsage {
+		data[k] = v
+		if w.printVerboseLogs {
+			framework.Logf("Get container %v usage on node %v. CPUUsageInCores: %v, MemoryUsageInBytes: %v, MemoryWorkingSetInBytes: %v", k, w.nodeName, v.CPUUsageInCores, v.MemoryUsageInBytes, v.MemoryWorkingSetInBytes)
 		}
 	}
 	w.dataSeries = append(w.dataSeries, data)
@@ -351,7 +334,6 @@ type ContainerResourceGatherer struct {

 // ResourceGathererOptions is a struct to hold options for resource.
 type ResourceGathererOptions struct {
-	InKubemark                  bool
 	Nodes                       NodesSet
 	ResourceDataGatheringPeriod time.Duration
 	ProbeDuration               time.Duration
@@ -402,20 +384,6 @@ func NewResourceUsageGatherer(ctx context.Context, c clientset.Interface, option
 		options:      options,
 	}

-	if options.InKubemark {
-		g.workerWg.Add(1)
-		g.workers = append(g.workers, resourceGatherWorker{
-			inKubemark:                  true,
-			stopCh:                      g.stopCh,
-			wg:                          &g.workerWg,
-			finished:                    false,
-			resourceDataGatheringPeriod: options.ResourceDataGatheringPeriod,
-			probeDuration:               options.ProbeDuration,
-			printVerboseLogs:            options.PrintVerboseLogs,
-		})
-		return &g, nil
-	}
-
 	// Tracks kube-system pods if no valid PodList is passed in.
 	var err error
 	if pods == nil {
@@ -475,7 +443,6 @@ func NewResourceUsageGatherer(ctx context.Context, c clientset.Interface, option
 				containerIDs:                g.containerIDs,
 				stopCh:                      g.stopCh,
 				finished:                    false,
-				inKubemark:                  false,
 				resourceDataGatheringPeriod: options.ResourceDataGatheringPeriod,
 				probeDuration:               options.ProbeDuration,
 				printVerboseLogs:            options.PrintVerboseLogs,
@@ -594,65 +561,3 @@ func (g *ContainerResourceGatherer) StopAndSummarize(percentiles []int, constrai
 	}
 	return &summary, nil
 }
-
-// kubemarkResourceUsage is a struct for tracking the resource usage of kubemark.
-type kubemarkResourceUsage struct {
-	Name                    string
-	MemoryWorkingSetInBytes uint64
-	CPUUsageInCores         float64
-}
-
-func getMasterUsageByPrefix(ctx context.Context, prefix string) (string, error) {
-	sshResult, err := e2essh.SSH(ctx, fmt.Sprintf("ps ax -o %%cpu,rss,command | tail -n +2 | grep %v | sed 's/\\s+/ /g'", prefix), framework.APIAddress()+":22", framework.TestContext.Provider)
-	if err != nil {
-		return "", err
-	}
-	return sshResult.Stdout, nil
-}
-
-// getKubemarkMasterComponentsResourceUsage returns the resource usage of kubemark which contains multiple combinations of cpu and memory usage for each pod name.
-func getKubemarkMasterComponentsResourceUsage(ctx context.Context) map[string]*kubemarkResourceUsage {
-	result := make(map[string]*kubemarkResourceUsage)
-	// Get kubernetes component resource usage
-	sshResult, err := getMasterUsageByPrefix(ctx, "kube")
-	if err != nil {
-		framework.Logf("Error when trying to SSH to master machine. Skipping probe. %v", err)
-		return nil
-	}
-	scanner := bufio.NewScanner(strings.NewReader(sshResult))
-	for scanner.Scan() {
-		var cpu float64
-		var mem uint64
-		var name string
-		fmt.Sscanf(strings.TrimSpace(scanner.Text()), "%f %d /usr/local/bin/kube-%s", &cpu, &mem, &name)
-		if name != "" {
-			// Gatherer expects pod_name/container_name format
-			fullName := name + "/" + name
-			result[fullName] = &kubemarkResourceUsage{Name: fullName, MemoryWorkingSetInBytes: mem * 1024, CPUUsageInCores: cpu / 100}
-		}
-	}
-	// Get etcd resource usage
-	sshResult, err = getMasterUsageByPrefix(ctx, "bin/etcd")
-	if err != nil {
-		framework.Logf("Error when trying to SSH to master machine. Skipping probe")
-		return nil
-	}
-	scanner = bufio.NewScanner(strings.NewReader(sshResult))
-	for scanner.Scan() {
-		var cpu float64
-		var mem uint64
-		var etcdKind string
-		fmt.Sscanf(strings.TrimSpace(scanner.Text()), "%f %d /bin/sh -c /usr/local/bin/etcd", &cpu, &mem)
-		dataDirStart := strings.Index(scanner.Text(), "--data-dir")
-		if dataDirStart < 0 {
-			continue
-		}
-		fmt.Sscanf(scanner.Text()[dataDirStart:], "--data-dir=/var/%s", &etcdKind)
-		if etcdKind != "" {
-			// Gatherer expects pod_name/container_name format
-			fullName := "etcd/" + etcdKind
-			result[fullName] = &kubemarkResourceUsage{Name: fullName, MemoryWorkingSetInBytes: mem * 1024, CPUUsageInCores: cpu / 100}
-		}
-	}
-	return result
-}
@@ -114,10 +114,9 @@ type Framework struct {
 	// test multiple times in parallel.
 	UniqueName string

-	clientConfig                     *rest.Config
-	restMapper                       *restmapper.DeferredDiscoveryRESTMapper
-	ClientSet                        clientset.Interface
-	KubemarkExternalClusterClientSet clientset.Interface
+	clientConfig *rest.Config
+	restMapper   *restmapper.DeferredDiscoveryRESTMapper
+	ClientSet    clientset.Interface

 	DynamicClient dynamic.Interface

@@ -31,7 +31,7 @@ func GrabBeforeEach(ctx context.Context, f *framework.Framework) (result *Collec
 	}

 	ginkgo.By("Gathering metrics before test", func() {
-		grabber, err := NewMetricsGrabber(ctx, f.ClientSet, f.KubemarkExternalClusterClientSet, f.ClientConfig(), !framework.ProviderIs("kubemark"), false, false, false, framework.TestContext.IncludeClusterAutoscalerMetrics, false)
+		grabber, err := NewMetricsGrabber(ctx, f.ClientSet, f.ClientConfig(), true, false, false, false, framework.TestContext.IncludeClusterAutoscalerMetrics, false)
 		if err != nil {
 			framework.Logf("Failed to create MetricsGrabber (skipping ClusterAutoscaler metrics gathering before test): %v", err)
 			return
@@ -55,8 +55,8 @@ func GrabAfterEach(ctx context.Context, f *framework.Framework, before *Collecti

 	ginkgo.By("Gathering metrics after test", func() {
 		// Grab apiserver, scheduler, controller-manager metrics and (optionally) nodes' kubelet metrics.
-		grabMetricsFromKubelets := framework.TestContext.GatherMetricsAfterTest != "master" && !framework.ProviderIs("kubemark")
-		grabber, err := NewMetricsGrabber(ctx, f.ClientSet, f.KubemarkExternalClusterClientSet, f.ClientConfig(), grabMetricsFromKubelets, true, true, true, framework.TestContext.IncludeClusterAutoscalerMetrics, false)
+		grabMetricsFromKubelets := framework.TestContext.GatherMetricsAfterTest != "master"
+		grabber, err := NewMetricsGrabber(ctx, f.ClientSet, f.ClientConfig(), grabMetricsFromKubelets, true, true, true, framework.TestContext.IncludeClusterAutoscalerMetrics, false)
 		if err != nil {
 			framework.Logf("Failed to create MetricsGrabber (skipping metrics gathering): %v", err)
 			return
@@ -118,7 +118,7 @@ func getKubeletMetricsFromNode(ctx context.Context, c clientset.Interface, nodeN
 	if c == nil {
 		return GrabKubeletMetricsWithoutProxy(ctx, nodeName, "/metrics")
 	}
-	grabber, err := NewMetricsGrabber(ctx, c, nil, nil, true, false, false, false, false, false)
+	grabber, err := NewMetricsGrabber(ctx, c, nil, true, false, false, false, false, false)
 	if err != nil {
 		return KubeletMetrics{}, err
 	}
@@ -70,7 +70,6 @@ type Collection struct {
 // Grabber provides functions which grab metrics from components
 type Grabber struct {
 	client                             clientset.Interface
-	externalClient                     clientset.Interface
 	config                             *rest.Config
 	grabFromAPIServer                  bool
 	grabFromControllerManager          bool
@@ -94,7 +93,7 @@ type Grabber struct {
 // Collecting metrics data is an optional debug feature. Not all clusters will
 // support it. If disabled for a component, the corresponding Grab function
 // will immediately return an error derived from MetricsGrabbingDisabledError.
-func NewMetricsGrabber(ctx context.Context, c clientset.Interface, ec clientset.Interface, config *rest.Config, kubelets bool, scheduler bool, controllers bool, apiServer bool, clusterAutoscaler bool, snapshotController bool) (*Grabber, error) {
+func NewMetricsGrabber(ctx context.Context, c clientset.Interface, config *rest.Config, kubelets bool, scheduler bool, controllers bool, apiServer bool, clusterAutoscaler bool, snapshotController bool) (*Grabber, error) {
 	kubeScheduler := ""
 	kubeControllerManager := ""
 	snapshotControllerManager := ""
@@ -128,13 +127,9 @@ func NewMetricsGrabber(ctx context.Context, c clientset.Interface, ec clientset.
 			break
 		}
 	}
-	if clusterAutoscaler && ec == nil {
-		klog.Warningf("Did not receive an external client interface. Grabbing metrics from ClusterAutoscaler is disabled.")
-	}

 	return &Grabber{
 		client:                     c,
-		externalClient:             ec,
 		config:                     config,
 		grabFromAPIServer:          apiServer,
 		grabFromControllerManager:  checkPodDebugHandlers(ctx, c, controllers, "kube-controller-manager", kubeControllerManager),
@@ -309,19 +304,10 @@ func (g *Grabber) GrabFromScheduler(ctx context.Context) (SchedulerMetrics, erro

 // GrabFromClusterAutoscaler returns metrics from cluster autoscaler
 func (g *Grabber) GrabFromClusterAutoscaler(ctx context.Context) (ClusterAutoscalerMetrics, error) {
-	if !g.HasControlPlanePods() && g.externalClient == nil {
+	if !g.HasControlPlanePods() {
 		return ClusterAutoscalerMetrics{}, fmt.Errorf("ClusterAutoscaler: %w", MetricsGrabbingDisabledError)
 	}
-	var client clientset.Interface
-	var namespace string
-	if g.externalClient != nil {
-		client = g.externalClient
-		namespace = "kubemark"
-	} else {
-		client = g.client
-		namespace = metav1.NamespaceSystem
-	}
-	output, err := g.getMetricsFromPod(ctx, client, "cluster-autoscaler", namespace, 8085)
+	output, err := g.getMetricsFromPod(ctx, g.client, "cluster-autoscaler", metav1.NamespaceSystem, 8085)
 	if err != nil {
 		return ClusterAutoscalerMetrics{}, err
 	}
@@ -1,9 +0,0 @@
-# This E2E framework sub-package is currently allowed to use arbitrary
-# dependencies, therefore we need to override the restrictions from
-# the parent .import-restrictions file.
-#
-# At some point it may become useful to also check this package's
-# dependencies more careful.
-rules:
-  - selectorRegexp: ""
-    allowedPrefixes: [ "" ]
@@ -1,97 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package kubemark
-
-import (
-	"flag"
-	"fmt"
-
-	"k8s.io/client-go/informers"
-	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/kubernetes/pkg/kubemark"
-	"k8s.io/kubernetes/test/e2e/framework"
-)
-
-var (
-	kubemarkExternalKubeConfig = flag.String(fmt.Sprintf("%s-%s", "kubemark-external", clientcmd.RecommendedConfigPathFlag), "", "Path to kubeconfig containing embedded authinfo for external cluster.")
-)
-
-func init() {
-	framework.RegisterProvider("kubemark", newProvider)
-}
-
-func newProvider() (framework.ProviderInterface, error) {
-	// Actual initialization happens when the e2e framework gets constructed.
-	return &Provider{}, nil
-}
-
-// Provider is a structure to handle Kubemark cluster for e2e testing
-type Provider struct {
-	framework.NullProvider
-	controller   *kubemark.KubemarkController
-	closeChannel chan struct{}
-}
-
-// ResizeGroup resizes an instance group
-func (p *Provider) ResizeGroup(group string, size int32) error {
-	return p.controller.SetNodeGroupSize(group, int(size))
-}
-
-// GetGroupNodes returns a node name for the specified node group
-func (p *Provider) GetGroupNodes(group string) ([]string, error) {
-	return p.controller.GetNodeNamesForNodeGroup(group)
-}
-
-// FrameworkBeforeEach prepares clients, configurations etc. for e2e testing
-func (p *Provider) FrameworkBeforeEach(f *framework.Framework) {
-	if *kubemarkExternalKubeConfig != "" && p.controller == nil {
-		externalConfig, err := clientcmd.BuildConfigFromFlags("", *kubemarkExternalKubeConfig)
-		externalConfig.QPS = f.Options.ClientQPS
-		externalConfig.Burst = f.Options.ClientBurst
-		framework.ExpectNoError(err)
-		externalClient, err := clientset.NewForConfig(externalConfig)
-		framework.ExpectNoError(err)
-		f.KubemarkExternalClusterClientSet = externalClient
-		p.closeChannel = make(chan struct{})
-		externalInformerFactory := informers.NewSharedInformerFactory(externalClient, 0)
-		kubemarkInformerFactory := informers.NewSharedInformerFactory(f.ClientSet, 0)
-		kubemarkNodeInformer := kubemarkInformerFactory.Core().V1().Nodes()
-		go kubemarkNodeInformer.Informer().Run(p.closeChannel)
-		p.controller, err = kubemark.NewKubemarkController(externalClient, externalInformerFactory, f.ClientSet, kubemarkNodeInformer)
-		framework.ExpectNoError(err)
-		externalInformerFactory.Start(p.closeChannel)
-		if !p.controller.WaitForCacheSync(p.closeChannel) {
-			framework.Failf("Unable to sync caches for %v", p.controller)
-		}
-		go p.controller.Run(p.closeChannel)
-	}
-}
-
-// FrameworkAfterEach cleans up after e2e testing
-func (p *Provider) FrameworkAfterEach(f *framework.Framework) {
-	if p.closeChannel != nil {
-		close(p.closeChannel)
-		p.controller = nil
-		p.closeChannel = nil
-	}
-}
-
-// GroupSize returns the size of an instance group
-func (p *Provider) GroupSize(group string) (int, error) {
-	return p.controller.GetNodeGroupSize(group)
-}
@@ -66,7 +66,7 @@ func GetSigner(provider string) (ssh.Signer, error) {
 	// support.
 	keyfile := ""
 	switch provider {
-	case "gce", "kubemark":
+	case "gce":
 		keyfile = os.Getenv("GCE_SSH_KEY")
 		if keyfile == "" {
 			keyfile = os.Getenv("GCE_SSH_PRIVATE_KEY_FILE")
@@ -36,14 +36,13 @@ import (
 var _ = common.SIGDescribe("Metrics", func() {
 	f := framework.NewDefaultFramework("metrics")
 	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
-	var c, ec clientset.Interface
+	var c clientset.Interface
 	var grabber *e2emetrics.Grabber
 	ginkgo.BeforeEach(func(ctx context.Context) {
 		var err error
 		c = f.ClientSet
-		ec = f.KubemarkExternalClusterClientSet
 		gomega.Eventually(ctx, func() error {
-			grabber, err = e2emetrics.NewMetricsGrabber(ctx, c, ec, f.ClientConfig(), true, true, true, true, true, true)
+			grabber, err = e2emetrics.NewMetricsGrabber(ctx, c, f.ClientConfig(), true, true, true, true, true, true)
 			return err
 		}, 5*time.Minute, 10*time.Second).Should(gomega.BeNil())
 	})
@@ -37,14 +37,13 @@ import (
 var _ = instrumentation.SIGDescribe("MetricsGrabber", func() {
 	f := framework.NewDefaultFramework("metrics-grabber")
 	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
-	var c, ec clientset.Interface
+	var c clientset.Interface
 	var grabber *e2emetrics.Grabber
 	ginkgo.BeforeEach(func(ctx context.Context) {
 		var err error
 		c = f.ClientSet
-		ec = f.KubemarkExternalClusterClientSet
 		gomega.Eventually(ctx, func() error {
-			grabber, err = e2emetrics.NewMetricsGrabber(ctx, c, ec, f.ClientConfig(), true, true, true, true, true, true)
+			grabber, err = e2emetrics.NewMetricsGrabber(ctx, c, f.ClientConfig(), true, true, true, true, true, true)
 			if err != nil {
 				return fmt.Errorf("failed to create metrics grabber: %w", err)
 			}
@@ -291,7 +291,7 @@ var _ = common.SIGDescribe("KubeProxy", func() {
 		nodeName := nodes.Items[0].Name

 		metricName := "kubeproxy_iptables_localhost_nodeports_accepted_packets_total"
-		metricsGrabber, err := e2emetrics.NewMetricsGrabber(ctx, fr.ClientSet, nil, fr.ClientConfig(), false, false, false, false, false, false)
+		metricsGrabber, err := e2emetrics.NewMetricsGrabber(ctx, fr.ClientSet, fr.ClientConfig(), false, false, false, false, false, false)
 		framework.ExpectNoError(err)

 		// create a pod with host-network for execing
@@ -21,7 +21,6 @@ import (
 	_ "k8s.io/kubernetes/test/e2e/framework/providers/aws"
 	_ "k8s.io/kubernetes/test/e2e/framework/providers/azure"
 	_ "k8s.io/kubernetes/test/e2e/framework/providers/gce"
-	_ "k8s.io/kubernetes/test/e2e/framework/providers/kubemark"
 	_ "k8s.io/kubernetes/test/e2e/framework/providers/openstack"
 	_ "k8s.io/kubernetes/test/e2e/framework/providers/vsphere"
 )
@@ -808,7 +808,7 @@ var _ = utils.SIGDescribe("CSI Mock selinux on mount metrics and SELinuxWarningC

 				// Some metrics use CSI driver name as a label, which is "csi-mock-" + the namespace name.
 				volumePluginLabel := "volume_plugin=\"kubernetes.io/csi/csi-mock-" + f.Namespace.Name + "\""
-				grabber, err := e2emetrics.NewMetricsGrabber(ctx, f.ClientSet, nil, f.ClientConfig(), true /*kubelet*/, false /*scheduler*/, true /*controllers*/, false /*apiserver*/, false /*autoscaler*/, false /*snapshotController*/)
+				grabber, err := e2emetrics.NewMetricsGrabber(ctx, f.ClientSet, f.ClientConfig(), true /*kubelet*/, false /*scheduler*/, true /*controllers*/, false /*apiserver*/, false /*autoscaler*/, false /*snapshotController*/)
 				framework.ExpectNoError(err, "creating the metrics grabber")

 				var nodeStageMountOpts, nodePublishMountOpts []string
@@ -310,7 +310,7 @@ var _ = utils.SIGDescribe("CSI Mock volume snapshot", func() {
 				}
 				ginkgo.DeferCleanup(m.cleanup)

-				metricsGrabber, err := e2emetrics.NewMetricsGrabber(ctx, m.config.Framework.ClientSet, nil, f.ClientConfig(), false, false, false, false, false, true)
+				metricsGrabber, err := e2emetrics.NewMetricsGrabber(ctx, m.config.Framework.ClientSet, f.ClientConfig(), false, false, false, false, false, true)
 				if err != nil {
 					framework.Failf("Error creating metrics grabber : %v", err)
 				}
@@ -125,7 +125,7 @@ func getVolumeOpCounts(ctx context.Context, c clientset.Interface, config *rest.

 	nodeLimit := 25

-	metricsGrabber, err := e2emetrics.NewMetricsGrabber(ctx, c, nil, config, true, false, true, false, false, false)
+	metricsGrabber, err := e2emetrics.NewMetricsGrabber(ctx, c, config, true, false, true, false, false, false)

 	if err != nil {
 		framework.ExpectNoError(err, "Error creating metrics grabber: %v", err)
@@ -124,7 +124,7 @@ var _ = utils.SIGDescribe("Volume metrics", func() {
 			VolumeMode:       ptr.To(v1.PersistentVolumeBlock),
 		}, ns)

-		metricsGrabber, err = e2emetrics.NewMetricsGrabber(ctx, c, nil, f.ClientConfig(), true, false, true, false, false, false)
+		metricsGrabber, err = e2emetrics.NewMetricsGrabber(ctx, c, f.ClientConfig(), true, false, true, false, false, false)

 		if err != nil {
 			framework.Failf("Error creating metrics grabber : %v", err)
@@ -54,8 +54,8 @@ func gatherTestSuiteMetrics(ctx context.Context) error {
 		return fmt.Errorf("error creating client: %w", err)
 	}

-	// Grab metrics for apiserver, scheduler, controller-manager, kubelet (for non-kubemark case) and cluster autoscaler (optionally).
-	grabber, err := e2emetrics.NewMetricsGrabber(ctx, c, nil, config, !framework.ProviderIs("kubemark"), true, true, true, framework.TestContext.IncludeClusterAutoscalerMetrics, false)
+	// Grab metrics for apiserver, scheduler, controller-manager, kubelet and cluster autoscaler (optionally).
+	grabber, err := e2emetrics.NewMetricsGrabber(ctx, c, config, true, true, true, true, framework.TestContext.IncludeClusterAutoscalerMetrics, false)
 	if err != nil {
 		return fmt.Errorf("failed to create MetricsGrabber: %w", err)
 	}
@@ -1,13 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-reviewers:
-  - sig-scalability-reviewers
-  - wojtek-t
-approvers:
-  - sig-scalability-approvers
-  - wojtek-t
-emeritus_approvers:
-  - gmarek
-  - shyamjvs
-labels:
-  - sig/scalability
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-CLOUD_PROVIDER="${CLOUD_PROVIDER:-gce}"
-CONTAINER_REGISTRY="${CONTAINER_REGISTRY:-gcr.io}"
-PROJECT="${PROJECT:-}"
-KUBEMARK_IMAGE_REGISTRY="${KUBEMARK_IMAGE_REGISTRY:-}"
-KUBEMARK_IMAGE_MAKE_TARGET="${KUBEMARK_IMAGE_MAKE_TARGET:-gcloudpush}"
@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Running cmd $RETRIES times in case of failures.
-function run-cmd-with-retries {
-  RETRIES="${RETRIES:-3}"
-  for attempt in $(seq 1 "${RETRIES}"); do
-    local ret_val=0
-    exec 5>&1 # Duplicate &1 to &5 for use below.
-    # We don't use 'local' to declare result as then ret_val always gets value 0.
-    # We use tee to output to &5 (redirected to stdout) while also storing it in the variable.
-    result=$("$@" 2>&1 | tee >(cat - >&5)) || ret_val="$?"
-    if [[ "${ret_val:-0}" -ne "0" ]]; then
-      if [[ $(echo "${result}" | grep -c "already exists") -gt 0 ]]; then
-        if [[ "${attempt}" == 1 ]]; then
-          # shellcheck disable=SC2154 # Color defined in sourced script
-          echo -e "${color_red}Failed to $1 $2 ${3:-} as the resource hasn't been deleted from a previous run.${color_norm}" >& 2
-          exit 1
-        fi
-        # shellcheck disable=SC2154 # Color defined in sourced script
-        echo -e "${color_yellow}Succeeded to $1 $2 ${3:-} in the previous attempt, but status response wasn't received.${color_norm}"
-        return 0
-      fi
-      # shellcheck disable=SC2154 # Color defined in sourced script
-      echo -e "${color_yellow}Attempt $attempt failed to $1 $2 ${3:-}. Retrying.${color_norm}" >& 2
-      sleep $((attempt * 5))
-    else
-      # shellcheck disable=SC2154 # Color defined in sourced script
-      echo -e "${color_green}Succeeded to $1 $2 ${3:-}.${color_norm}"
-      return 0
-    fi
-  done
-  # shellcheck disable=SC2154 # Color defined in sourced script
-  echo -e "${color_red}Failed to $1 $2 ${3:-}.${color_norm}" >& 2
-  exit 1
-}
@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script assumes that kubectl binary is present in PATH.
-kubectl config set-cluster hollow-cluster --server=http://localhost:8080 --insecure-skip-tls-verify=true
-kubectl config set-credentials "$(whoami)"
-kubectl config set-context hollow-context --cluster=hollow-cluster --user="$(whoami)"
@@ -1,137 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../..
-
-source "${KUBE_ROOT}/test/kubemark/common/util.sh"
-
-# Wrapper for gcloud compute, running it $RETRIES times in case of failures.
-# Args:
-# $@: all stuff that goes after 'gcloud compute'
-function run-gcloud-compute-with-retries {
-  run-cmd-with-retries gcloud compute "$@"
-}
-
-function authenticate-docker {
-  echo "Configuring registry authentication"
-  mkdir -p "${HOME}/.docker"
-  gcloud beta auth configure-docker -q
-}
-
-function create-kubemark-master {
-  # We intentionally override env vars in subshell to preserve original values.
-  # shellcheck disable=SC2030,SC2031
-  (
-    # All calls to e2e-grow-cluster must share temp dir with initial e2e-up.sh.
-    kube::util::ensure-temp-dir
-    export KUBE_TEMP="${KUBE_TEMP}"
-
-    export KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark"
-    export KUBECONFIG_INTERNAL="${RESOURCE_DIRECTORY}/kubeconfig-internal.kubemark"
-    export CLUSTER_NAME="${CLUSTER_NAME}-kubemark"
-    export KUBE_CREATE_NODES=false
-    export KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX}-kubemark"
-
-    # Quite tricky cidr setup: we set KUBE_GCE_ENABLE_IP_ALIASES=true to avoid creating
-    # cloud routes and RangeAllocator to assign cidrs by kube-controller-manager.
-    export KUBE_GCE_ENABLE_IP_ALIASES=true
-    export KUBE_GCE_NODE_IPAM_MODE=RangeAllocator
-
-    # Disable all addons. They are running outside of the kubemark cluster.
-    export KUBE_ENABLE_CLUSTER_AUTOSCALER=false
-    export KUBE_ENABLE_CLUSTER_DNS=false
-    export KUBE_ENABLE_NODE_LOGGING=false
-    export KUBE_ENABLE_METRICS_SERVER=false
-    export KUBE_ENABLE_L7_LOADBALANCING="none"
-
-    # Unset env variables set by kubetest for 'root cluster'. We need recompute them
-    # for kubemark master.
-    # TODO(mborsz): Figure out some better way to filter out such env variables than
-    # listing them here.
-    unset MASTER_SIZE MASTER_DISK_SIZE MASTER_ROOT_DISK_SIZE
-
-    # Set kubemark-specific overrides:
-    # for each defined env KUBEMARK_X=Y call export X=Y.
-    for var in ${!KUBEMARK_*}; do
-      dst_var=${var#KUBEMARK_}
-      val=${!var}
-      echo "Setting ${dst_var} to '${val}'"
-      export "${dst_var}"="${val}"
-    done
-
-    "${KUBE_ROOT}/hack/e2e-internal/e2e-up.sh"
-
-    if [[ "${KUBEMARK_HA_MASTER:-}" == "true" && -n "${KUBEMARK_MASTER_ADDITIONAL_ZONES:-}" ]]; then
-        for KUBE_GCE_ZONE in ${KUBEMARK_MASTER_ADDITIONAL_ZONES}; do
-          KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_REPLICATE_EXISTING_MASTER=true \
-            "${KUBE_ROOT}/hack/e2e-internal/e2e-grow-cluster.sh"
-        done
-    fi
-
-    # The e2e-up.sh script is not sourced, so we don't have access to variables that
-    # it sets. Instead, we read data which was written to the KUBE_TEMP directory.
-    # The cluster-location is either ZONE (say us-east1-a) or REGION (say us-east1).
-    # To get REGION from location, only first two parts are matched.
-    REGION=$(grep -o "^[a-z]*-[a-z0-9]*" "${KUBE_TEMP}"/cluster-location.txt)
-    MASTER_NAME="${KUBE_GCE_INSTANCE_PREFIX}"-master
-
-    if [[ ${GCE_PRIVATE_CLUSTER:-} == "true" ]]; then
-      MASTER_INTERNAL_IP=$(gcloud compute addresses describe "${MASTER_NAME}-internal-ip" \
-          --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
-    fi
-    MASTER_IP=$(gcloud compute addresses describe "${MASTER_NAME}-ip" \
-        --project "${PROJECT}" --region "${REGION}" -q --format='value(address)')
-
-    # If cluster uses private master IP, two kubeconfigs are created:
-    # - kubeconfig with public IP, which will be used to connect to the cluster
-    #     from outside of the cluster network
-    # - kubeconfig with private IP (called internal kubeconfig), which will be
-    #      used to create hollow nodes.
-    #
-    # Note that hollow nodes might use either of these kubeconfigs, but
-    # using internal one is better from performance and cost perspective, since
-    # traffic does not need to go through Cloud NAT.
-    if [[ -n "${MASTER_INTERNAL_IP:-}" ]]; then
-      echo "Writing internal kubeconfig to '${KUBECONFIG_INTERNAL}'"
-      ip_regexp=${MASTER_IP//./\\.} # escape ".", so that sed won't treat it as "any char"
-      sed "s/${ip_regexp}/${MASTER_INTERNAL_IP}/g" "${KUBECONFIG}" > "${KUBECONFIG_INTERNAL}"
-    fi
-    )
-}
-
-function delete-kubemark-master {
-  # We intentionally override env vars in subshell to preserve original values.
-  # shellcheck disable=SC2030,SC2031
-  (
-    export CLUSTER_NAME="${CLUSTER_NAME}-kubemark"
-    export KUBE_GCE_INSTANCE_PREFIX="${KUBE_GCE_INSTANCE_PREFIX}-kubemark"
-
-    export KUBE_DELETE_NETWORK=false
-
-    if [[ "${KUBEMARK_HA_MASTER:-}" == "true" && -n "${KUBEMARK_MASTER_ADDITIONAL_ZONES:-}" ]]; then
-      for KUBE_GCE_ZONE in ${KUBEMARK_MASTER_ADDITIONAL_ZONES}; do
-        KUBE_GCE_ZONE="${KUBE_GCE_ZONE}" KUBE_REPLICATE_EXISTING_MASTER=true \
-          "${KUBE_ROOT}/hack/e2e-internal/e2e-shrink-cluster.sh"
-      done
-    fi
-
-    "${KUBE_ROOT}/hack/e2e-internal/e2e-down.sh"
-  )
-}
-
-function calculate-node-labels {
-  echo "cloud.google.com/metadata-proxy-ready=true"
-}
@@ -1,54 +0,0 @@
-# Kubemark Pre-existing Provider Guide
-
-**Kubemark Master**
- A set of Kubernetes control plane components running in a VM
-
-**Kubernetes Cluster**
- A real Kubernetes Cluster that has master and nodes. The hollow-node pods
-  are run in this cluster, but appear as nodes to the Kubemark Master
-
-## Introduction
-
-Every running Kubemark setup looks like the following:
- 1) A running Kubernetes cluster pointed to by the local kubeconfig
- 2) A separate VM where the kubemark master is running
- 3) Some hollow-nodes that run on the Kubernetes Cluster from #1
- 4) The hollow-nodes are configured to talk with the kubemark master at #2
-
-When using the pre-existing provider, the developer is responsible for creating
-#1 and #2.  Therefore, the kubemark scripts will not create any infrastructure
-or start a kubemark master like in other providers. Instead, the existing
-resources provided by the VM at $MASTER_IP will serve as the kubemark master.
-
-## Use Case
-
-The goal of the pre-existing provider is to use the kubemark tools with an
-existing kubemark master. It's meant to provide the developer with
-additional flexibility to customize the cluster infrastructure and still use
-the kubemark setup tools.  The pre-existing provider is an **advanced** use
-case that requires the developer to have knowledge of setting up a kubemark
-master.
-
-## Requirements
-
-To use the pre-existing provider, the expectation is that there's a kubemark
-master that is reachable at $MASTER_IP. The machine that the kubemark master is
-on has to be ssh able from the host that's executing the kubemark scripts. And
-the user on that machine has to be 'kubernetes'.
-
-Requirement checklist:
- Set MASTER_IP to ip address to the kubemark master
- The host where you execute the kubemark scripts must be able to ssh to
-  kubernetes@$MASTER_IP
-
-## Example Configuration
-
-_test/kubemark/cloud-provider-config.sh_
-
-```
-CLOUD_PROVIDER="pre-existing"
-KUBEMARK_IMAGE_MAKE_TARGET="push"
-CONTAINER_REGISTRY=docker.io
-PROJECT="rthallisey"
-MASTER_IP="192.168.121.29:6443"
-```
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../..
-
-source "${KUBE_ROOT}/test/kubemark/common/util.sh"
-
-# Leave the skeleton definition of execute-cmd-on-master-with-retries
-# so only the pre-existing provider functions will target this.
-function execute-cmd-on-pre-existing-master-with-retries() {
-  IP_WITHOUT_PORT=$(echo "${MASTER_IP}" | cut -f 1 -d ':') || "${MASTER_IP}"
-
-  RETRIES="${2:-1}" run-cmd-with-retries ssh kubernetes@"${IP_WITHOUT_PORT}" "${1}"
-}
@@ -1,94 +0,0 @@
-{
-    "kind": "Pod",
-    "apiVersion": "v1",
-    "metadata": {
-        "name": "cluster-autoscaler",
-        "namespace": "kubemark",
-        "labels": {
-            "tier": "cluster-management",
-            "component": "cluster-autoscaler"
-        }
-    },
-    "spec": {
-        "hostNetwork": true,
-        "containers": [
-            {
-                "name": "cluster-autoscaler",
-                "image": "registry.k8s.io/cluster-autoscaler:v1.0.0",
-                "command": [
-                    "./run.sh",
-                    "--kubernetes=https://{{master_ip}}:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/cluster_autoscaler.kubeconfig",
-                    "--v=4",
-                    "--logtostderr=true",
-                    "--write-status-configmap=true",
-                    "--cloud-provider=kubemark",
-                    "--nodes={{kubemark_autoscaler_min_nodes}}:{{kubemark_autoscaler_max_nodes}}:{{kubemark_autoscaler_mig_name}}"
-                ],
-                "env": [
-                    {
-                        "name": "LOG_OUTPUT",
-                        "value": "/var/log/cluster-autoscaler.log"
-                    }
-                ],
-                "resources": {
-                    "requests": {
-                        "cpu": "10m",
-                        "memory": "300Mi"
-                    }
-                },
-                "volumeMounts": [
-                    {"name": "cloudconfigmount","mountPath": "/etc/gce.conf", "readOnly": true},
-                    {
-                        "name": "ssl-certs",
-                        "readOnly": true,
-                        "mountPath": "/etc/ssl/certs"
-                    },
-                    {
-                        "name": "usrsharecacerts",
-                        "readOnly": true,
-                        "mountPath": "/usr/share/ca-certificates"
-                    },
-                    {
-                        "name": "logdir",
-                        "mountPath": "/var/log",
-                        "readOnly": false
-                    },
-                    {
-                        "name": "kubeconfig-volume",
-                        "mountPath": "/kubeconfig"
-                    }
-                ],
-                "terminationMessagePath": "/dev/termination-log",
-                "imagePullPolicy": "Always"
-            }
-        ],
-        "volumes": [
-            {"name": "cloudconfigmount","hostPath": {"path": "/etc/gce.conf"}},
-            {
-                "name": "ssl-certs",
-                "hostPath": {
-                    "path": "/etc/ssl/certs"
-                }
-            },
-            {
-                "name": "usrsharecacerts",
-                "hostPath": {
-                    "path": "/usr/share/ca-certificates"
-                }
-            },
-            {
-                "name": "logdir",
-                "hostPath": {
-                    "path": "/var/log"
-                }
-            },
-            {
-                "name": "kubeconfig-volume",
-                "secret": {
-                    "secretName": "kubeconfig"
-                }
-            }
-        ],
-        "restartPolicy": "Always"
-    }
-}
@@ -1,82 +0,0 @@
-{
-	"kind": "ReplicationController",
-	"apiVersion": "v1",
-	"metadata": {
-		"name": "heapster-v1.3.0",
-		"labels": {
-			"k8s-app": "heapster",
-			"version": "v1.3.0"
-		}
-	},
-	"spec": {
-		"replicas": 1,
-		"selector": {
-			"k8s-app": "heapster",
-			"version": "v1.3.0"
-		},
-		"template": {
-			"metadata": {
-				"labels": {
-					"k8s-app": "heapster",
-					"version": "v1.3.0"
-				}
-			},
-			"spec": {
-				"volumes": [
-					{
-						"name": "kubeconfig-volume",
-						"secret": {
-							"secretName": "kubeconfig"
-						}
-					}
-				],
-				"containers": [
-				{
-					"name": "heapster",
-					"image": "registry.k8s.io/heapster:v1.3.0",
-					"resources": {
-						"requests": {
-							"cpu": "{{METRICS_CPU}}m",
-							"memory": "{{METRICS_MEM}}Mi"
-						}
-					},
-					"command": [
-						"/heapster"
-					],
-					"args": [
-						"--source=kubernetes:https://{{MASTER_IP}}:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/heapster.kubeconfig"
-					],
-					"volumeMounts": [
-						{
-							"name": "kubeconfig-volume",
-							"mountPath": "/kubeconfig"
-						}
-					]
-				},
-				{
-					"name": "eventer",
-					"image": "registry.k8s.io/heapster:v1.3.0",
-					"resources": {
-						"requests": {
-							"memory": "{{EVENTER_MEM}}Ki"
-						}
-					},
-					"command": [
-						"/eventer"
-					],
-					"args": [
-						"--source=kubernetes:https://{{MASTER_IP}}:443?inClusterConfig=0&useServiceAccount=0&auth=/kubeconfig/heapster.kubeconfig"
-					],
-					"volumeMounts": [
-						{
-							"name": "kubeconfig-volume",
-							"mountPath": "/kubeconfig"
-						}
-					]
-
-				}]
-			}
-		}
-	}
-}
-
@@ -1,138 +0,0 @@
-apiVersion: v1
-kind: ReplicationController
-metadata:
-  name: hollow-node
-  labels:
-    name: hollow-node
-    {{kubemark_mig_config}}
-spec:
-  replicas: {{numreplicas}}
-  selector:
-    name: hollow-node
-  template:
-    metadata:
-      labels:
-        name: hollow-node
-        {{kubemark_mig_config}}
-    spec:
-      initContainers:
-      - name: init-inotify-limit
-        image: busybox:1.32
-        command: ['sysctl', '-w', 'fs.inotify.max_user_instances=1000']
-        securityContext:
-          privileged: true
-      volumes:
-      - name: kubeconfig-volume
-        secret:
-          secretName: kubeconfig
-      - name: kernelmonitorconfig-volume
-        configMap:
-          name: node-configmap
-      - name: logs-volume
-        hostPath:
-          path: /var/log
-      - name: containerd
-        hostPath:
-          path: /run/containerd
-      - name: no-serviceaccount-access-to-real-master
-        emptyDir: {}
-      containers:
-      - name: hollow-kubelet
-        image: {{kubemark_image_registry}}/kubemark:{{kubemark_image_tag}}
-        ports:
-        - containerPort: 4194
-        - containerPort: 10250
-        - containerPort: 10255
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        command: [
-          "/go-runner",
-          "-log-file=/var/log/kubelet-$(NODE_NAME).log",
-          "/kubemark",
-          "--morph=kubelet",
-          "--name=$(NODE_NAME)",
-          "--kubeconfig=/kubeconfig/kubelet.kubeconfig",
-          "--node-labels={{hollow_node_labels}}",
-          {{hollow_kubelet_params}}
-        ]
-        volumeMounts:
-        - name: kubeconfig-volume
-          mountPath: /kubeconfig
-          readOnly: true
-        - name: logs-volume
-          mountPath: /var/log
-        - name: containerd
-          mountPath: /run/containerd
-        resources:
-          requests:
-            cpu: {{hollow_kubelet_millicpu}}m
-            memory: {{hollow_kubelet_mem_Ki}}Ki
-        securityContext:
-          privileged: true
-      - name: hollow-proxy
-        image: {{kubemark_image_registry}}/kubemark:{{kubemark_image_tag}}
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        command: [
-          "/go-runner",
-          "-log-file=/var/log/kubeproxy-$(NODE_NAME).log",
-          "/kubemark",
-          "--morph=proxy",
-          "--name=$(NODE_NAME)",
-          "--kubeconfig=/kubeconfig/kubeproxy.kubeconfig",
-          {{hollow_proxy_params}}
-        ]
-        volumeMounts:
-        - name: kubeconfig-volume
-          mountPath: /kubeconfig
-          readOnly: true
-        - name: logs-volume
-          mountPath: /var/log
-        resources:
-          requests:
-            cpu: {{hollow_proxy_millicpu}}m
-            memory: {{hollow_proxy_mem_Ki}}Ki
-      - name: hollow-node-problem-detector
-        image: registry.k8s.io/node-problem-detector/node-problem-detector:v1.35.2
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        command:
-        - /bin/sh
-        - -c
-        - /node-problem-detector --system-log-monitors=/config/kernel.monitor --apiserver-override="https://{{master_ip}}:443?inClusterConfig=false&auth=/kubeconfig/npd.kubeconfig" --alsologtostderr 1>>/var/log/npd-$(NODE_NAME).log 2>&1
-        volumeMounts:
-        - name: kubeconfig-volume
-          mountPath: /kubeconfig
-          readOnly: true
-        - name: kernelmonitorconfig-volume
-          mountPath: /config
-          readOnly: true
-        - name: no-serviceaccount-access-to-real-master
-          mountPath: /var/run/secrets/kubernetes.io/serviceaccount
-          readOnly: true
-        - name: logs-volume
-          mountPath: /var/log
-        resources:
-          requests:
-            cpu: {{npd_millicpu}}m
-            memory: {{npd_mem_Ki}}Ki
-        securityContext:
-          privileged: true
-      # Keep the pod running on unreachable node for 15 minutes.
-      # This time should be sufficient for a VM reboot and should
-      # avoid recreating a new hollow node.
-      # See https://github.com/kubernetes/kubernetes/issues/67120 for context.
-      tolerations:
-      - key: "node.kubernetes.io/unreachable"
-        operator: "Exists"
-        effect: "NoExecute"
-        tolerationSeconds: 900
@@ -1,20 +0,0 @@
-{
-	"plugin": "filelog",
-	"pluginConfig": {
-		"timestamp": "dummy",
-		"message": "dummy",
-		"timestampFormat": "dummy"
-	},
-	"logPath": "/dev/null",
-	"lookback": "10m",
-	"bufferSize": 10,
-	"source": "kernel-monitor",
-	"conditions": [
-		{
-			"type": "KernelDeadlock",
-			"reason": "KernelHasNoDeadlock",
-			"message": "kernel has no deadlock"
-		}
-	],
-	"rules": []
-}
@@ -1,187 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: kube-dns
-  namespace: kubemark
-  labels:
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: Reconcile
-    kubernetes.io/name: "KubeDNS"
-spec:
-  selector:
-    k8s-app: kube-dns
-  ports:
-  - name: dns
-    port: 53
-    protocol: UDP
-  - name: dns-tcp
-    port: 53
-    protocol: TCP
---
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-dns
-  namespace: kubemark
-  labels:
-    kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: Reconcile
---
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kube-dns
-  namespace: kubemark
-  labels:
-    addonmanager.kubernetes.io/mode: EnsureExists
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: kube-dns
-  namespace: kubemark
-  labels:
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: Reconcile
-spec:
-  strategy:
-    rollingUpdate:
-      maxSurge: 10%
-      maxUnavailable: 0
-  selector:
-    matchLabels:
-      k8s-app: kube-dns
-  template:
-    metadata:
-      labels:
-        k8s-app: kube-dns
-    spec:
-      priorityClassName: system-node-critical
-      tolerations:
-      - key: "CriticalAddonsOnly"
-        operator: "Exists"
-      volumes:
-      - name: kube-dns-config
-        configMap:
-          name: kube-dns
-          optional: true
-      - name: secret-volume
-        secret:
-          secretName: kubeconfig
-      containers:
-      - name: kubedns
-        image: registry.k8s.io/k8s-dns-kube-dns-amd64:1.14.9
-        resources:
-          limits:
-            memory: 170Mi
-          requests:
-            cpu: 100m
-            memory: 70Mi
-        livenessProbe:
-          httpGet:
-            path: /healthcheck/kubedns
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        readinessProbe:
-          httpGet:
-            path: /readiness
-            port: 8081
-            scheme: HTTP
-          initialDelaySeconds: 3
-          timeoutSeconds: 5
-        args:
-        - --domain={{dns_domain}}.
-        - --dns-port=10053
-        - --config-dir=/kube-dns-config
-        - --kubecfg-file=/etc/secret-volume/dns.kubeconfig
-        - --v=2
-        env:
-        - name: PROMETHEUS_PORT
-          value: "10055"
-        ports:
-        - containerPort: 10053
-          name: dns-local
-          protocol: UDP
-        - containerPort: 10053
-          name: dns-tcp-local
-          protocol: TCP
-        - containerPort: 10055
-          name: metrics
-          protocol: TCP
-        volumeMounts:
-        - name: kube-dns-config
-          mountPath: /kube-dns-config
-        - name: secret-volume
-          mountPath: /etc/secret-volume
-      - name: dnsmasq
-        image: registry.k8s.io/k8s-dns-dnsmasq-nanny-amd64:1.14.9
-        livenessProbe:
-          httpGet:
-            path: /healthcheck/dnsmasq
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        args:
-        - -v=2
-        - -logtostderr
-        - -configDir=/etc/k8s/dns/dnsmasq-nanny
-        - -restartDnsmasq=true
-        - --
-        - -k
-        - --cache-size=1000
-        - --no-negcache
-        - --dns-loop-detect
-        - --log-facility=-
-        - --server=/{{dns_domain}}/127.0.0.1#10053
-        - --server=/in-addr.arpa/127.0.0.1#10053
-        - --server=/ip6.arpa/127.0.0.1#10053
-        ports:
-        - containerPort: 53
-          name: dns
-          protocol: UDP
-        - containerPort: 53
-          name: dns-tcp
-          protocol: TCP
-        # see: https://github.com/kubernetes/kubernetes/issues/29055 for details
-        resources:
-          requests:
-            cpu: 150m
-            memory: 20Mi
-        volumeMounts:
-        - name: kube-dns-config
-          mountPath: /etc/k8s/dns/dnsmasq-nanny
-      - name: sidecar
-        image: registry.k8s.io/k8s-dns-sidecar-amd64:1.14.9
-        livenessProbe:
-          httpGet:
-            path: /metrics
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        args:
-        - --v=2
-        - --logtostderr
-        - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.{{dns_domain}},5,SRV
-        - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.{{dns_domain}},5,SRV
-        ports:
-        - containerPort: 10054
-          name: metrics
-          protocol: TCP
-        resources:
-          requests:
-            memory: 20Mi
-            cpu: 10m
-      dnsPolicy: Default  # Don't use cluster DNS.
-      serviceAccountName: kube-dns
@@ -1,7 +0,0 @@
-{
-	"kind": "Namespace",
-	"apiVersion": "v1",
-	"metadata": {
-		"name": "kubemark"
-	}
-}
@@ -1 +0,0 @@
-These resources are used to add extra (non-default) bindings to kubemark to match users and groups that are particular to the kubemark environment. These are not standard bootstrap bindings and not standard users they are bound to, and have been adapted from cluster/addons/e2e-rbac-bindings. Tighten/loosen these access rights as required in future.
@@ -1,16 +0,0 @@
-# This is the role binding for the kubemark cluster autoscaler.
-# TODO: Use proper Cluster Autoscaler role (github.com/kubernetes/autoscaler/issues/383)
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: cluster-autoscaler-view-binding
-  labels:
-    kubernetes.io/cluster-service: "true"
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: cluster-admin
-subjects:
- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: system:cluster-autoscaler
@@ -1,15 +0,0 @@
-# This is the role binding for the kubemark heapster.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: heapster-view-binding
-  labels:
-    kubernetes.io/cluster-service: "true"
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:heapster
-subjects:
- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: system:heapster
@@ -1,15 +0,0 @@
-# This is the role binding for the kubemark kube-dns.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kube-dns-view-binding
-  labels:
-    kubernetes.io/cluster-service: "true"
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-dns
-subjects:
- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: system:kube-dns
@@ -1,18 +0,0 @@
-# This is the role binding for the local kubectl, which is
-# used for listing hollow-nodes in start-kubemark.sh and
-# send resource creation requests, etc in run-e2e-tests.sh.
-# Also useful if you manually want to use local kubectl.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kubecfg-cluster-admin
-  labels:
-    kubernetes.io/cluster-service: "true"
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: cluster-admin
-subjects:
- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: kubecfg
@@ -1,18 +0,0 @@
-# The Kubemark environment currently gives all kubelets a single shared credential.
-#
-# TODO: give each kubelet a credential in the system:nodes group with username system:node:<nodeName>,
-# to exercise the Node authorizer and admission, then remove this binding
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kubelet-node
-  labels:
-    kubernetes.io/cluster-service: "true"
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:node
-subjects:
- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: kubelet
@@ -1,15 +0,0 @@
-# This is the role binding for the node-problem-detector.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: node-problem-detector-binding
-  labels:
-    kubernetes.io/cluster-service: "true"
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:node-problem-detector
-subjects:
- apiGroup: rbac.authorization.k8s.io
-  kind: User
-  name: system:node-problem-detector
@@ -1,43 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-addon-manager
-  namespace: kube-system
-  labels:
-    component: kube-addon-manager
-spec:
-  hostNetwork: true
-  containers:
-  - name: kube-addon-manager
-    image: {{kube_docker_registry}}/addon-manager/kube-addon-manager:v9.1.8
-    command:
-    - /bin/bash
-    - -c
-    - /opt/kube-addons-main.sh 1>>/var/log/kube-addon-manager.log 2>&1
-    resources:
-      requests:
-        cpu: 5m
-        memory: 50Mi
-    volumeMounts:
-    - name: addons
-      mountPath: /etc/kubernetes/
-      readOnly: true
-    - name: varlog
-      mountPath: /var/log/kube-addon-manager.log
-    - mountPath: /etc/srv/kubernetes/addon-manager/
-      name: srvkube
-      readOnly: true
-    env:
-    - name: KUBECTL_OPTS
-      value: '--kubeconfig=/etc/srv/kubernetes/addon-manager/kubeconfig'
-  volumes:
-  - name: addons
-    hostPath:
-      path: /etc/kubernetes/
-  - name: varlog
-    hostPath:
-      path: /var/log/kube-addon-manager.log
-      type: FileOrCreate
-  - hostPath:
-      path: /etc/srv/kubernetes/addon-manager/
-    name: srvkube
@@ -1,776 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Script that starts kubelet on kubemark-master as a supervisord process
-# and then runs the master components as pods using kubelet.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# Define key path variables.
-KUBE_ROOT="/home/kubernetes"
-KUBE_BINDIR="${KUBE_ROOT}/kubernetes/server/bin"
-
-function config-ip-firewall {
-  echo "Configuring IP firewall rules"
-  # The GCI image has host firewall which drop most inbound/forwarded packets.
-  # We need to add rules to accept all TCP/UDP/ICMP packets.
-  if iptables -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then
-    echo "Add rules to accept all inbound TCP/UDP/ICMP packets"
-    iptables -A INPUT -w -p TCP -j ACCEPT
-    iptables -A INPUT -w -p UDP -j ACCEPT
-    iptables -A INPUT -w -p ICMP -j ACCEPT
-  fi
-  if iptables -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then
-    echo "Add rules to accept all forwarded TCP/UDP/ICMP packets"
-    iptables -A FORWARD -w -p TCP -j ACCEPT
-    iptables -A FORWARD -w -p UDP -j ACCEPT
-    iptables -A FORWARD -w -p ICMP -j ACCEPT
-  fi
-}
-
-function create-dirs {
-	echo "Creating required directories"
-	mkdir -p /var/lib/kubelet
-	mkdir -p /etc/kubernetes/manifests
-	mkdir -p /etc/kubernetes/addons
-}
-
-# Setup working directory for kubelet.
-function setup-kubelet-dir {
-	echo "Making /var/lib/kubelet executable for kubelet"
-	mount -B /var/lib/kubelet /var/lib/kubelet/
-	mount -B -o remount,exec,suid,dev /var/lib/kubelet
-}
-
-# Remove any default etcd config dirs/files.
-function delete-default-etcd-configs {
-	if [[ -d /etc/etcd ]]; then
-		rm -rf /etc/etcd
-	fi
-	if [[ -e /etc/default/etcd ]]; then
-		rm -f /etc/default/etcd
-	fi
-	if [[ -e /etc/systemd/system/etcd.service ]]; then
-		rm -f /etc/systemd/system/etcd.service
-	fi
-	if [[ -e /etc/init.d/etcd ]]; then
-		rm -f /etc/init.d/etcd
-	fi
-}
-
-# Compute etcd related variables.
-function compute-etcd-variables {
-	ETCD_IMAGE="${ETCD_IMAGE:-}"
-	ETCD_QUOTA_BYTES=""
-	if [ "${ETCD_VERSION:0:2}" == "3." ]; then
-		# TODO: Set larger quota to see if that helps with
-		# 'mvcc: database space exceeded' errors. If so, pipe
-		# though our setup scripts.
-		ETCD_QUOTA_BYTES=" --quota-backend-bytes=4294967296 "
-	fi
-}
-
-# Formats the given device ($1) if needed and mounts it at given mount point
-# ($2).
-function safe-format-and-mount() {
-	device=$1
-	mountpoint=$2
-
-	# Format only if the disk is not already formatted.
-	if ! tune2fs -l "${device}" ; then
-		echo "Formatting '${device}'"
-		mkfs.ext4 -F "${device}"
-	fi
-
-	echo "Mounting '${device}' at '${mountpoint}'"
-	mount -o discard,defaults "${device}" "${mountpoint}"
-}
-
-# Finds a PD device with name '$1' attached to the master.
-function find-attached-pd() {
-	local -r pd_name=$1
-	if [[ ! -e /dev/disk/by-id/${pd_name} ]]; then
-		echo ""
-	fi
-	device_info=$(ls -l "/dev/disk/by-id/${pd_name}")
-	relative_path=${device_info##* }
-	echo "/dev/disk/by-id/${relative_path}"
-}
-
-# Mounts a persistent disk (formatting if needed) to store the persistent data
-# on the master. safe-format-and-mount only formats an unformatted disk, and
-# mkdir -p will leave a directory be if it already exists.
-function mount-pd() {
-	local -r pd_name=$1
-	local -r mount_point=$2
-
-	if [[ -z "${find-attached-pd ${pd_name}}" ]]; then
-		echo "Can't find ${pd_name}. Skipping mount."
-		return
-	fi
-
-	local -r pd_path="/dev/disk/by-id/${pd_name}"
-	echo "Mounting PD '${pd_path}' at '${mount_point}'"
-	# Format and mount the disk, create directories on it for all of the master's
-	# persistent data, and link them to where they're used.
-	mkdir -p "${mount_point}"
-	safe-format-and-mount "${pd_path}" "${mount_point}"
-	echo "Mounted PD '${pd_path}' at '${mount_point}'"
-
-	# NOTE: These locations on the PD store persistent data, so to maintain
-	# upgradeability, these locations should not change.  If they do, take care
-	# to maintain a migration path from these locations to whatever new
-	# locations.
-}
-
-# Create kubeconfig for controller-manager's service account authentication.
-function create-kubecontrollermanager-kubeconfig {
-	echo "Creating kube-controller-manager kubeconfig file"
-	mkdir -p "${KUBE_ROOT}/k8s_auth_data/kube-controller-manager"
-	cat <<EOF >"${KUBE_ROOT}/k8s_auth_data/kube-controller-manager/kubeconfig"
-apiVersion: v1
-kind: Config
-users:
- name: kube-controller-manager
-  user:
-    token: ${KUBE_CONTROLLER_MANAGER_TOKEN}
-clusters:
- name: local
-  cluster:
-    insecure-skip-tls-verify: true
-    server: https://localhost:443
-contexts:
- context:
-    cluster: local
-    user: kube-controller-manager
-  name: service-account-context
-current-context: service-account-context
-EOF
-}
-
-function create-kubescheduler-kubeconfig {
-  echo "Creating kube-scheduler kubeconfig file"
-  mkdir -p "${KUBE_ROOT}/k8s_auth_data/kube-scheduler"
-  cat <<EOF >"${KUBE_ROOT}/k8s_auth_data/kube-scheduler/kubeconfig"
-apiVersion: v1
-kind: Config
-users:
- name: kube-scheduler
-  user:
-    token: ${KUBE_SCHEDULER_TOKEN}
-clusters:
- name: local
-  cluster:
-    insecure-skip-tls-verify: true
-    server: https://localhost:443
-contexts:
- context:
-    cluster: local
-    user: kube-scheduler
-  name: kube-scheduler
-current-context: kube-scheduler
-EOF
-}
-
-function create-addonmanager-kubeconfig {
-  echo "Creating addonmanager kubeconfig file"
-  mkdir -p "${KUBE_ROOT}/k8s_auth_data/addon-manager"
-  cat <<EOF >"${KUBE_ROOT}/k8s_auth_data/addon-manager/kubeconfig"
-apiVersion: v1
-kind: Config
-users:
- name: addon-manager
-  user:
-    token: ${ADDON_MANAGER_TOKEN}
-clusters:
- name: local
-  cluster:
-    insecure-skip-tls-verify: true
-    server: https://localhost:443
-contexts:
- context:
-    cluster: local
-    user: addon-manager
-  name: addon-manager
-current-context: addon-manager
-EOF
-}
-
-function assemble-docker-flags {
-	echo "Assemble docker command line flags"
-	local docker_opts="-p /var/run/docker.pid --iptables=false --ip-masq=false"
-	docker_opts+=" --log-level=debug"  # Since it's a test cluster
-	# TODO(shyamjvs): Incorporate network plugin options, etc later.
-	echo "DOCKER_OPTS=\"${docker_opts}\"" > /etc/default/docker
-	echo "DOCKER_NOFILE=65536" >> /etc/default/docker  # For setting ulimit -n
-	systemctl restart docker
-}
-
-# A helper function for loading a docker image. It keeps trying up to 5 times.
-#
-# $1: Full path of the docker image
-function try-load-docker-image {
-	local -r img=$1
-	echo "Try to load docker image file ${img}"
-	# Temporarily turn off errexit, because we don't want to exit on first failure.
-	set +e
-	local -r max_attempts=5
-	local -i attempt_num=1
-	until timeout 30 docker load -i "${img}"; do
-		if [[ "${attempt_num}" == "${max_attempts}" ]]; then
-			echo "Fail to load docker image file ${img} after ${max_attempts} retries. Exit!!"
-			exit 1
-		else
-			attempt_num=$((attempt_num+1))
-			sleep 5
-		fi
-	done
-	# Re-enable errexit.
-	set -e
-}
-
-# Loads kube-system docker images. It is better to do it before starting kubelet,
-# as kubelet will restart docker daemon, which may interfere with loading images.
-function load-docker-images {
-	echo "Start loading kube-system docker images"
-	local -r img_dir="${KUBE_BINDIR}"
-	try-load-docker-image "${img_dir}/kube-apiserver.tar"
-	try-load-docker-image "${img_dir}/kube-controller-manager.tar"
-	try-load-docker-image "${img_dir}/kube-scheduler.tar"
-}
-
-# Computes command line arguments to be passed to kubelet.
-function compute-kubelet-params {
-	local params="${KUBELET_TEST_ARGS:-}"
-	params+=" --cgroup-root=/"
-	params+=" --cloud-provider=gce"
-	params+=" --pod-manifest-path=/etc/kubernetes/manifests"
-	if [[ -n "${KUBELET_PORT:-}" ]]; then
-		params+=" --port=${KUBELET_PORT}"
-	fi
-	params+=" --enable-debugging-handlers=false"
-	params+=" --hairpin-mode=none"
-	echo "${params}"
-}
-
-# Creates the systemd config file for kubelet.service.
-function create-kubelet-conf() {
-	local -r kubelet_bin="$1"
-	local -r kubelet_env_file="/etc/default/kubelet"
-	local -r flags=$(compute-kubelet-params)
-	echo "KUBELET_OPTS=\"${flags}\"" > "${kubelet_env_file}"
-
-	# Write the systemd service file for kubelet.
-	cat <<EOF >/etc/systemd/system/kubelet.service
-[Unit]
-Description=Kubermark kubelet
-Requires=network-online.target
-After=network-online.target
-
-[Service]
-Restart=always
-RestartSec=10
-EnvironmentFile=${kubelet_env_file}
-ExecStart=${kubelet_bin} \$KUBELET_OPTS
-
-[Install]
-WantedBy=multi-user.target
-EOF
-}
-
-# This function assembles the kubelet systemd service file and starts it using
-# systemctl, on the kubemark master.
-function start-kubelet {
-	# Create systemd config.
-	local -r kubelet_bin="/usr/bin/kubelet"
-	create-kubelet-conf "${kubelet_bin}"
-
-	# Flush iptables nat table
-  	iptables -t nat -F || true
-
-	# Start the kubelet service.
-	systemctl start kubelet.service
-}
-
-# Create the log file and set its properties.
-#
-# $1 is the file to create.
-function prepare-log-file {
-	touch "$1"
-	chmod 644 "$1"
-	chown root:root "$1"
-}
-
-# A helper function for copying addon manifests and set dir/files
-# permissions.
-#
-# $1: addon category under /etc/kubernetes
-# $2: manifest source dir
-function setup-addon-manifests {
-  local -r src_dir="${KUBE_ROOT}/$2"
-  local -r dst_dir="/etc/kubernetes/$1/$2"
-
-  if [[ ! -d "${dst_dir}" ]]; then
-    mkdir -p "${dst_dir}"
-  fi
-
-  local files
-  files=$(find "${src_dir}" -maxdepth 1 -name "*.yaml")
-  if [[ -n "${files}" ]]; then
-    cp "${src_dir}/"*.yaml "${dst_dir}"
-  fi
-  chown -R root:root "${dst_dir}"
-  chmod 755 "${dst_dir}"
-  chmod 644 "${dst_dir}"/*
-}
-
-# Write the config for the audit policy.
-# Note: This duplicates the function in cluster/gce/gci/configure-helper.sh.
-# TODO: Get rid of this function when #53321 is fixed.
-function create-master-audit-policy {
-  local -r path="${1}"
-  local -r policy="${2:-}"
-
-  if [[ -n "${policy}" ]]; then
-    echo "${policy}" > "${path}"
-    return
-  fi
-
-  # Known api groups
-  local -r known_apis='
-      - group: "" # core
-      - group: "admissionregistration.k8s.io"
-      - group: "apiextensions.k8s.io"
-      - group: "apiregistration.k8s.io"
-      - group: "apps"
-      - group: "authentication.k8s.io"
-      - group: "authorization.k8s.io"
-      - group: "autoscaling"
-      - group: "batch"
-      - group: "certificates.k8s.io"
-      - group: "extensions"
-      - group: "metrics"
-      - group: "networking.k8s.io"
-      - group: "policy"
-      - group: "rbac.authorization.k8s.io"
-      - group: "storage.k8s.io"'
-
-  cat <<EOF >"${path}"
-apiVersion: audit.k8s.io/v1
-kind: Policy
-rules:
-  # The following requests were manually identified as high-volume and low-risk,
-  # so drop them.
-  - level: None
-    users: ["system:kube-proxy"]
-    verbs: ["watch"]
-    resources:
-      - group: "" # core
-        resources: ["endpoints", "services", "services/status"]
-  - level: None
-    # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
-    # TODO(#46983): Change this to the ingress controller service account.
-    users: ["system:unsecured"]
-    namespaces: ["kube-system"]
-    verbs: ["get"]
-    resources:
-      - group: "" # core
-        resources: ["configmaps"]
-  - level: None
-    users: ["kubelet"] # legacy kubelet identity
-    verbs: ["get"]
-    resources:
-      - group: "" # core
-        resources: ["nodes", "nodes/status"]
-  - level: None
-    userGroups: ["system:nodes"]
-    verbs: ["get"]
-    resources:
-      - group: "" # core
-        resources: ["nodes", "nodes/status"]
-  - level: None
-    users:
-      - system:kube-controller-manager
-      - system:kube-scheduler
-      - system:serviceaccount:kube-system:endpoint-controller
-    verbs: ["get", "update"]
-    namespaces: ["kube-system"]
-    resources:
-      - group: "" # core
-        resources: ["endpoints"]
-  - level: None
-    users: ["system:apiserver"]
-    verbs: ["get"]
-    resources:
-      - group: "" # core
-        resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
-  # Don't log HPA fetching metrics.
-  - level: None
-    users:
-      - system:kube-controller-manager
-    verbs: ["get", "list"]
-    resources:
-      - group: "metrics"
-  # Don't log these read-only URLs.
-  - level: None
-    nonResourceURLs:
-      - /healthz*
-      - /version
-      - /swagger*
-  # Don't log events requests.
-  - level: None
-    resources:
-      - group: "" # core
-        resources: ["events"]
-  # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
-  - level: Request
-    users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"]
-    verbs: ["update","patch"]
-    resources:
-      - group: "" # core
-        resources: ["nodes/status", "pods/status"]
-    omitStages:
-      - "RequestReceived"
-  - level: Request
-    userGroups: ["system:nodes"]
-    verbs: ["update","patch"]
-    resources:
-      - group: "" # core
-        resources: ["nodes/status", "pods/status"]
-    omitStages:
-      - "RequestReceived"
-  # deletecollection calls can be large, don't log responses for expected namespace deletions
-  - level: Request
-    users: ["system:serviceaccount:kube-system:namespace-controller"]
-    verbs: ["deletecollection"]
-    omitStages:
-      - "RequestReceived"
-  # Secrets, ConfigMaps, TokenRequest and TokenReviews can contain sensitive & binary data,
-  # so only log at the Metadata level.
-  - level: Metadata
-    resources:
-      - group: "" # core
-        resources: ["secrets", "configmaps", "serviceaccounts/token"]
-      - group: authentication.k8s.io
-        resources: ["tokenreviews"]
-    omitStages:
-      - "RequestReceived"
-  # Get responses can be large; skip them.
-  - level: Request
-    verbs: ["get", "list", "watch"]
-    resources: ${known_apis}
-    omitStages:
-      - "RequestReceived"
-  # Default level for known APIs
-  - level: RequestResponse
-    resources: ${known_apis}
-    omitStages:
-      - "RequestReceived"
-  # Default level for all other requests.
-  - level: Metadata
-    omitStages:
-      - "RequestReceived"
-EOF
-}
-
-# Computes command line arguments to be passed to etcd.
-function compute-etcd-params {
-	local params="${ETCD_TEST_ARGS:-}"
-	params+=" --name=etcd-$(hostname -s)"
-	params+=" --listen-peer-urls=http://127.0.0.1:2380"
-	params+=" --advertise-client-urls=http://127.0.0.1:2379"
-	params+=" --listen-client-urls=http://0.0.0.0:2379"
-
-	# Enable apiserver->etcd auth.
-	params+=" --client-cert-auth"
-	params+=" --trusted-ca-file /etc/srv/kubernetes/etcd-apiserver-ca.crt"
-	params+=" --cert-file /etc/srv/kubernetes/etcd-apiserver-server.crt"
-	params+=" --key-file /etc/srv/kubernetes/etcd-apiserver-server.key"
-
-	params+=" --data-dir=/var/etcd/data"
-	params+=" ${ETCD_QUOTA_BYTES}"
-	echo "${params}"
-}
-
-# Computes command line arguments to be passed to etcd-events.
-function compute-etcd-events-params {
-	local params="${ETCD_TEST_ARGS:-}"
-	params+=" --name=etcd-$(hostname -s)"
-	params+=" --listen-peer-urls=http://127.0.0.1:2381"
-	params+=" --advertise-client-urls=http://127.0.0.1:4002"
-	params+=" --listen-client-urls=http://0.0.0.0:4002"
-	params+=" --data-dir=/var/etcd/data-events"
-	params+=" ${ETCD_QUOTA_BYTES}"
-	echo "${params}"
-}
-
-# Computes command line arguments to be passed to apiserver.
-function compute-kube-apiserver-params {
-	local params="--insecure-bind-address=0.0.0.0"
-	params+=" --etcd-servers=${ETCD_SERVERS:-http://127.0.0.1:2379}"
-	if [[ -z "${ETCD_SERVERS:-}" ]]; then
-		params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#${EVENT_STORE_URL}}"
-	elif [[ -n "${ETCD_SERVERS_OVERRIDES:-}" ]]; then
-		params+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}"
-	fi
-	# Enable apiserver->etcd auth.
-	params+=" --etcd-cafile=/etc/srv/kubernetes/etcd-apiserver-ca.crt"
-	params+=" --etcd-certfile=/etc/srv/kubernetes/etcd-apiserver-client.crt"
-	params+=" --etcd-keyfile=/etc/srv/kubernetes/etcd-apiserver-client.key"
-
-	params+=" --tls-cert-file=/etc/srv/kubernetes/server.cert"
-	params+=" --tls-private-key-file=/etc/srv/kubernetes/server.key"
-	params+=" --requestheader-client-ca-file=/etc/srv/kubernetes/aggr_ca.crt"
-	params+=" --requestheader-allowed-names=aggregator"
-	params+=" --requestheader-extra-headers-prefix=X-Remote-Extra-"
-	params+=" --requestheader-group-headers=X-Remote-Group"
-	params+=" --requestheader-username-headers=X-Remote-User"
-	params+=" --proxy-client-cert-file=/etc/srv/kubernetes/proxy_client.crt"
-	params+=" --proxy-client-key-file=/etc/srv/kubernetes/proxy_client.key"
-	params+=" --enable-aggregator-routing=true"
-	params+=" --client-ca-file=/etc/srv/kubernetes/ca.crt"
-	params+=" --token-auth-file=/etc/srv/kubernetes/known_tokens.csv"
-	params+=" --secure-port=443"
-	params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
-	params+=" --admission-control=${CUSTOM_ADMISSION_PLUGINS}"
-	params+=" --authorization-mode=Node,RBAC"
-	params+=" --allow-privileged=true"
-	if [[ -n "${STORAGE_BACKEND:-}" ]]; then
-		params+=" --storage-backend=${STORAGE_BACKEND}"
-	fi
-	if [[ -n "${STORAGE_MEDIA_TYPE:-}" ]]; then
-		params+=" --storage-media-type=${STORAGE_MEDIA_TYPE}"
-	fi
-  if [[ -n "${ETCD_COMPACTION_INTERVAL_SEC:-}" ]]; then
-    params+=" --etcd-compaction-interval=${ETCD_COMPACTION_INTERVAL_SEC}s"
-  fi
-	if [[ -n "${KUBE_APISERVER_REQUEST_TIMEOUT:-}" ]]; then
-		params+=" --min-request-timeout=${KUBE_APISERVER_REQUEST_TIMEOUT}"
-	fi
-	if [[ "${NUM_NODES}" -ge 3000 ]]; then
-		params+=" --max-requests-inflight=3000 --max-mutating-requests-inflight=1000"
-	elif [[ "${NUM_NODES}" -ge 1000 ]]; then
-		params+=" --max-requests-inflight=1500 --max-mutating-requests-inflight=500"
-	fi
-	if [[ -n "${RUNTIME_CONFIG:-}" ]]; then
-		params+=" --runtime-config=${RUNTIME_CONFIG}"
-	fi
-	if [[ -n "${FEATURE_GATES:-}" ]]; then
-		params+=" --feature-gates=${FEATURE_GATES}"
-	fi
-	if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then
-		# Create the audit policy file, and mount it into the apiserver pod.
-		create-master-audit-policy "${audit_policy_file}" "${ADVANCED_AUDIT_POLICY:-}"
-
-		# The config below matches the one in cluster/gce/gci/configure-helper.sh.
-		# TODO: Currently supporting just log backend. Support webhook if needed.
-		params+=" --audit-policy-file=${audit_policy_file}"
-		params+=" --audit-log-path=/var/log/kube-apiserver-audit.log"
-		params+=" --audit-log-maxage=0"
-		params+=" --audit-log-maxbackup=0"
-		params+=" --audit-log-maxsize=2000000000"
-	fi
-        # Append APISERVER_TEST_ARGS to the end, which will allow for
-        # the above defaults to be overridden.
-	params+=" ${APISERVER_TEST_ARGS:-}"
-	echo "${params}"
-}
-
-# Computes command line arguments to be passed to controller-manager.
-function compute-kube-controller-manager-params {
-	local params="${CONTROLLER_MANAGER_TEST_ARGS:-}"
-	params+=" --use-service-account-credentials"
-	params+=" --kubeconfig=/etc/srv/kubernetes/kube-controller-manager/kubeconfig"
-	params+=" --service-account-private-key-file=/etc/srv/kubernetes/server.key"
-	params+=" --root-ca-file=/etc/srv/kubernetes/ca.crt"
-	params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}"
-	params+=" --cluster-cidr=${CLUSTER_IP_RANGE}"
-	params+=" --service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}"
-	params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
-	echo "${params}"
-}
-
-# Computes command line arguments to be passed to scheduler.
-function compute-kube-scheduler-params {
-	local params="${SCHEDULER_TEST_ARGS:-}"
-	params+=" --kubeconfig=/etc/srv/kubernetes/kube-scheduler/kubeconfig"
-	echo "${params}"
-}
-
-# Computes command line arguments to be passed to addon-manager.
-function compute-kube-addon-manager-params {
-	echo ""
-}
-
-# Start a kubernetes master component '$1' which can be any of the following:
-# 1. etcd
-# 2. etcd-events
-# 3. kube-apiserver
-# 4. kube-controller-manager
-# 5. kube-scheduler
-# 6. kube-addon-manager
-#
-# It prepares the log file, loads the docker tag, calculates variables, sets them
-# in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
-#
-# Assumed vars:
-#   DOCKER_REGISTRY
-function start-kubemaster-component() {
-	echo "Start master component $1"
-	local -r component=$1
-	prepare-log-file /var/log/"${component}".log
-	local -r src_file="${KUBE_ROOT}/${component}.yaml"
-	local -r params=$("compute-${component}-params")
-
-	# Evaluate variables.
-	sed -i -e "s@{{params}}@${params}@g" "${src_file}"
-	sed -i -e "s@{{kube_docker_registry}}@${DOCKER_REGISTRY}@g" "${src_file}"
-	sed -i -e "s@{{instance_prefix}}@${INSTANCE_PREFIX}@g" "${src_file}"
-	if [ "${component:0:4}" == "etcd" ]; then
-		sed -i -e "s@{{etcd_image}}@${ETCD_IMAGE}@g" "${src_file}"
-	elif [ "${component}" == "kube-addon-manager" ]; then
-		setup-addon-manifests "addons" "kubemark-rbac-bindings"
-	else
-		local -r component_docker_tag=$(cat "${KUBE_BINDIR}/${component}.docker_tag")
-		sed -i -e "s@{{${component}_docker_tag}}@${component_docker_tag}@g" "${src_file}"
-		if [ "${component}" == "kube-apiserver" ]; then
-			local audit_policy_config_mount=""
-			local audit_policy_config_volume=""
-			if [[ "${ENABLE_APISERVER_ADVANCED_AUDIT:-}" == "true" ]]; then
-				read -r -d '' audit_policy_config_mount << EOF
- name: auditpolicyconfigmount
-  mountPath: ${audit_policy_file}
-  readOnly: true
-EOF
-				read -r -d '' audit_policy_config_volume << EOF
- name: auditpolicyconfigmount
-  hostPath:
-    path: ${audit_policy_file}
-    type: FileOrCreate
-EOF
-			fi
-			sed -i -e "s@{{audit_policy_config_mount}}@${audit_policy_config_mount}@g" "${src_file}"
-			sed -i -e "s@{{audit_policy_config_volume}}@${audit_policy_config_volume}@g" "${src_file}"
-		fi
-	fi
-	cp "${src_file}" /etc/kubernetes/manifests
-}
-
-############################### Main Function ########################################
-echo "Start to configure master instance for kubemark"
-
-# Extract files from the server tar and setup master env variables.
-cd "${KUBE_ROOT}"
-if [[ ! -d "${KUBE_ROOT}/kubernetes" ]]; then
-	tar xzf kubernetes-server-linux-amd64.tar.gz
-fi
-source "${KUBE_ROOT}/kubemark-master-env.sh"
-
-# Setup IP firewall rules, required directory structure and etcd config.
-config-ip-firewall
-create-dirs
-setup-kubelet-dir
-delete-default-etcd-configs
-compute-etcd-variables
-
-# Setup authentication tokens and kubeconfigs for kube-controller-manager and kube-scheduler,
-# only if their kubeconfigs don't already exist as this script could be running on reboot.
-if [[ ! -f "${KUBE_ROOT}/k8s_auth_data/kube-controller-manager/kubeconfig" ]]; then
-	KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | LC_CTYPE=C tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
-	echo "${KUBE_CONTROLLER_MANAGER_TOKEN},system:kube-controller-manager,uid:system:kube-controller-manager" >> "${KUBE_ROOT}/k8s_auth_data/known_tokens.csv"
-	create-kubecontrollermanager-kubeconfig
-fi
-if [[ ! -f "${KUBE_ROOT}/k8s_auth_data/kube-scheduler/kubeconfig" ]]; then
-	KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | LC_CTYPE=C tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
-	echo "${KUBE_SCHEDULER_TOKEN},system:kube-scheduler,uid:system:kube-scheduler" >> "${KUBE_ROOT}/k8s_auth_data/known_tokens.csv"
-	create-kubescheduler-kubeconfig
-fi
-
-ADDON_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | LC_CTYPE=C tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
-echo "${ADDON_MANAGER_TOKEN},system:addon-manager,admin,system:masters" >> "${KUBE_ROOT}/k8s_auth_data/known_tokens.csv"
-create-addonmanager-kubeconfig
-
-# Mount master PD for etcd and create symbolic links to it.
-{
-	main_etcd_mount_point="/mnt/disks/master-pd"
-	mount-pd "google-master-pd" "${main_etcd_mount_point}"
-	# Contains all the data stored in etcd.
-	mkdir -p "${main_etcd_mount_point}/var/etcd"
-	chmod 700 "${main_etcd_mount_point}/var/etcd"
-	ln -s -f "${main_etcd_mount_point}/var/etcd" /var/etcd
-	mkdir -p /etc/srv
-	# Setup the dynamically generated apiserver auth certs and keys to pd.
-	mkdir -p "${main_etcd_mount_point}/srv/kubernetes"
-	ln -s -f "${main_etcd_mount_point}/srv/kubernetes" /etc/srv/kubernetes
-	# Copy the files to the PD only if they don't exist (so we do it only the first time).
-	if [[ "$(ls -A ${main_etcd_mount_point}/srv/kubernetes/)" == "" ]]; then
-		cp -r "${KUBE_ROOT}"/k8s_auth_data/* "${main_etcd_mount_point}/srv/kubernetes/"
-	fi
-	# Directory for kube-apiserver to store SSH key (if necessary).
-	mkdir -p "${main_etcd_mount_point}/srv/sshproxy"
-	ln -s -f "${main_etcd_mount_point}/srv/sshproxy" /etc/srv/sshproxy
-}
-
-# Mount master PD for event-etcd (if required) and create symbolic links to it.
-{
-	EVENT_STORE_IP="${EVENT_STORE_IP:-127.0.0.1}"
-	EVENT_STORE_URL="${EVENT_STORE_URL:-http://${EVENT_STORE_IP}:4002}"
-	if [ "${EVENT_PD:-}" == "true" ]; then
-		event_etcd_mount_point="/mnt/disks/master-event-pd"
-		mount-pd "google-master-event-pd" "${event_etcd_mount_point}"
-		# Contains all the data stored in event etcd.
-		mkdir -p "${event_etcd_mount_point}/var/etcd/events"
-		chmod 700 "${event_etcd_mount_point}/var/etcd/events"
-		ln -s -f "${event_etcd_mount_point}/var/etcd/events" /var/etcd/events
-	fi
-}
-
-# Setup docker flags and load images of the master components.
-assemble-docker-flags
-DOCKER_REGISTRY="registry.k8s.io"
-load-docker-images
-
-readonly audit_policy_file="/etc/audit_policy.config"
-
-# Start kubelet as a supervisord process and master components as pods.
-start-kubelet
-if [[ -z "${ETCD_SERVERS:-}" ]]; then
-	start-kubemaster-component "etcd"
-	if [ "${EVENT_STORE_IP:-}" == "127.0.0.1" ]; then
-		start-kubemaster-component "etcd-events"
-	fi
-fi
-start-kubemaster-component "kube-apiserver"
-start-kubemaster-component "kube-controller-manager"
-start-kubemaster-component "kube-scheduler"
-start-kubemaster-component "kube-addon-manager"
-
-# Wait till apiserver is working fine or timeout.
-echo -n "Waiting for apiserver to be healthy"
-start=$(date +%s)
-until [ "$(curl 127.0.0.1:8080/healthz 2> /dev/null)" == "ok" ]; do
-	echo -n "."
-	sleep 1
-	now=$(date +%s)
-	if [ $((now - start)) -gt 300 ]; then
-		echo "Timeout!"
-		exit 1
-	fi
-done
-
-echo "Done for the configuration for kubemark master"
@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-export KUBERNETES_PROVIDER="kubemark"
-export KUBE_CONFIG_FILE="config-default.sh"
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
-
-# We need an absolute path to KUBE_ROOT
-ABSOLUTE_ROOT=$(readlink -f "${KUBE_ROOT}")
-
-source "${KUBE_ROOT}/cluster/kubemark/util.sh"
-
-echo "Kubemark master name: ${MASTER_NAME}"
-echo "Kubemark master ip: ${KUBE_MASTER_IP}"
-
-export KUBE_MASTER_URL="https://${KUBE_MASTER_IP}"
-export KUBECONFIG="${ABSOLUTE_ROOT}/test/kubemark/resources/kubeconfig.kubemark"
-export E2E_MIN_STARTUP_PODS=0
-
-ARGS=("$@")
-
-if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER}" == "true" ]]; then
-  ARGS+=("--kubemark-external-kubeconfig=${DEFAULT_KUBECONFIG}")
-fi
-
-# Running locally.
-for ((i=0; i < ${#ARGS[@]}; i++)); do
-  ARGS[i]="$(echo "${ARGS[$i]}" | sed -e 's/\[/\\\[/g' -e 's/\]/\\\]/g' )"
-done
-"${KUBE_ROOT}/hack/ginkgo-e2e.sh" "--e2e-verify-service-account=false" "--dump-logs-on-failure=false" "${ARGS[@]}"
@@ -1,62 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2017 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script contains the helper functions that each provider hosting
-# Kubermark must implement to use test/kubemark/start-kubemark.sh and
-# test/kubemark/stop-kubemark.sh scripts.
-
-# This function should authenticate docker to be able to read/write to
-# the right container registry (needed for pushing kubemark image).
-function authenticate-docker {
-	echo "Configuring registry authentication" 1>&2
-}
-
-# This function should create kubemark master and write kubeconfig to
-# "${RESOURCE_DIRECTORY}/kubeconfig.kubemark".
-# If a cluster uses private master IP, create-kubemark-master might also write
-# a second kubeconfig to "${RESOURCE_DIRECTORY}/kubeconfig-internal.kubemark".
-# The difference between these two kubeconfigs is that the internal one uses
-# private master IP, which might be better suited for setting up hollow nodes.
-function create-kubemark-master {
-  echo "Creating cluster..."
-}
-
-# This function should delete kubemark master.
-function delete-kubemark-master {
-  echo "Deleting cluster..."
-}
-
-# This function should return node labels.
-function calculate-node-labels {
-  echo ""
-}
-
-# Common colors used throughout the kubemark scripts
-if [[ -z "${color_start-}" ]]; then
-  declare -r color_start="\033["
-  # shellcheck disable=SC2034
-  declare -r color_red="${color_start}0;31m"
-  # shellcheck disable=SC2034
-  declare -r color_yellow="${color_start}0;33m"
-  # shellcheck disable=SC2034
-  declare -r color_green="${color_start}0;32m"
-  # shellcheck disable=SC2034
-  declare -r color_blue="${color_start}1;34m"
-  # shellcheck disable=SC2034
-  declare -r color_cyan="${color_start}1;36m"
-  # shellcheck disable=SC2034
-  declare -r color_norm="${color_start}0m"
-fi
@@ -1,255 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Script that creates a Kubemark cluster for any given cloud provider.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-TMP_ROOT="$(dirname "${BASH_SOURCE[@]}")/../.."
-KUBE_ROOT=$(readlink -e "${TMP_ROOT}" 2> /dev/null || perl -MCwd -e 'print Cwd::abs_path shift' "${TMP_ROOT}")
-
-source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh"
-source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
-source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh"
-source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"
-
-if [[ -f "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/startup.sh" ]] ; then
-  source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/startup.sh"
-fi
-
-source "${KUBE_ROOT}/cluster/kubemark/util.sh"
-
-KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
-KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
-RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
-LOCAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig.kubemark"
-INTERNAL_KUBECONFIG="${RESOURCE_DIRECTORY}/kubeconfig-internal.kubemark"
-
-# Generate a random 6-digit alphanumeric tag for the kubemark image.
-# Used to uniquify image builds across different invocations of this script.
-KUBEMARK_IMAGE_TAG=$(head /dev/urandom | LC_CTYPE=C tr -dc 'a-z0-9' | fold -w 6 | head -n 1)
-
-# Create a docker image for hollow-node and upload it to the appropriate docker registry.
-function create-and-upload-hollow-node-image {
-  authenticate-docker
-  KUBEMARK_IMAGE_REGISTRY="${KUBEMARK_IMAGE_REGISTRY:-${CONTAINER_REGISTRY}/${PROJECT}}"
-  # Build+push the image through makefile.
-  build_cmd=("make" "${KUBEMARK_IMAGE_MAKE_TARGET}")
-  MAKE_DIR="${KUBE_ROOT}/cluster/images/kubemark"
-  KUBEMARK_BIN="$(kube::util::find-binary-for-platform kubemark linux/amd64)"
-  if [[ -z "${KUBEMARK_BIN}" ]]; then
-    echo 'Cannot find cmd/kubemark binary'
-    exit 1
-  fi
-  echo "Copying kubemark binary to ${MAKE_DIR}"
-  cp "${KUBEMARK_BIN}" "${MAKE_DIR}"
-  CURR_DIR=$(pwd)
-  cd "${MAKE_DIR}"
-  REGISTRY=${KUBEMARK_IMAGE_REGISTRY} IMAGE_TAG=${KUBEMARK_IMAGE_TAG} run-cmd-with-retries "${build_cmd[@]}"
-  rm kubemark
-  cd "$CURR_DIR"
-  echo "Created and uploaded the kubemark hollow-node image to docker registry."
-  # Cleanup the kubemark image after the script exits.
-  if [[ "${CLEANUP_KUBEMARK_IMAGE:-}" == "true" ]]; then
-    trap delete-kubemark-image EXIT
-  fi
-}
-
-function delete-kubemark-image {
-  delete-image "${KUBEMARK_IMAGE_REGISTRY}/kubemark:${KUBEMARK_IMAGE_TAG}"
-}
-
-# Generate secret and configMap for the hollow-node pods to work, prepare
-# manifests of the hollow-node and heapster replication controllers from
-# templates, and finally create these resources through kubectl.
-function create-kube-hollow-node-resources {
-  # Create kubemark namespace.
-  "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/kubemark-ns.json"
-
-  # Create configmap for configuring hollow- kubelet, proxy and npd.
-  "${KUBECTL}" create configmap "node-configmap" --namespace="kubemark" \
-    --from-file=kernel.monitor="${RESOURCE_DIRECTORY}/kernel-monitor.json"
-
-  # Create secret for passing kubeconfigs to kubelet, kubeproxy and npd.
-  # It's bad that all component shares the same kubeconfig.
-  # TODO(https://github.com/kubernetes/kubernetes/issues/79883): Migrate all components to separate credentials.
-  "${KUBECTL}" create secret generic "kubeconfig" --type=Opaque --namespace="kubemark" \
-    --from-file=kubelet.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
-    --from-file=kubeproxy.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
-    --from-file=npd.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
-    --from-file=heapster.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
-    --from-file=cluster_autoscaler.kubeconfig="${HOLLOWNODE_KUBECONFIG}" \
-    --from-file=dns.kubeconfig="${HOLLOWNODE_KUBECONFIG}"
-
-  # Create addon pods.
-  # Heapster.
-  mkdir -p "${RESOURCE_DIRECTORY}/addons"
-  sed "s@{{MASTER_IP}}@${MASTER_IP}@g" "${RESOURCE_DIRECTORY}/heapster_template.json" > "${RESOURCE_DIRECTORY}/addons/heapster.json"
-  metrics_mem_per_node=4
-  metrics_mem=$((200 + metrics_mem_per_node*NUM_NODES))
-  sed -i'' -e "s@{{METRICS_MEM}}@${metrics_mem}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
-  metrics_cpu_per_node_numerator=${NUM_NODES}
-  metrics_cpu_per_node_denominator=2
-  metrics_cpu=$((80 + metrics_cpu_per_node_numerator / metrics_cpu_per_node_denominator))
-  sed -i'' -e "s@{{METRICS_CPU}}@${metrics_cpu}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
-  eventer_mem_per_node=500
-  eventer_mem=$((200 * 1024 + eventer_mem_per_node*NUM_NODES))
-  sed -i'' -e "s@{{EVENTER_MEM}}@${eventer_mem}@g" "${RESOURCE_DIRECTORY}/addons/heapster.json"
-
-  # Cluster Autoscaler.
-  if [[ "${ENABLE_KUBEMARK_CLUSTER_AUTOSCALER:-}" == "true" ]]; then
-    echo "Setting up Cluster Autoscaler"
-    KUBEMARK_AUTOSCALER_MIG_NAME="${KUBEMARK_AUTOSCALER_MIG_NAME:-${NODE_INSTANCE_PREFIX}-group}"
-    KUBEMARK_AUTOSCALER_MIN_NODES="${KUBEMARK_AUTOSCALER_MIN_NODES:-0}"
-    KUBEMARK_AUTOSCALER_MAX_NODES="${KUBEMARK_AUTOSCALER_MAX_NODES:-10}"
-    NUM_NODES=${KUBEMARK_AUTOSCALER_MAX_NODES}
-    echo "Setting maximum cluster size to ${NUM_NODES}."
-    KUBEMARK_MIG_CONFIG="autoscaling.k8s.io/nodegroup: ${KUBEMARK_AUTOSCALER_MIG_NAME}"
-    sed "s/{{master_ip}}/${MASTER_IP}/g" "${RESOURCE_DIRECTORY}/cluster-autoscaler_template.json" > "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
-    sed -i'' -e "s@{{kubemark_autoscaler_mig_name}}@${KUBEMARK_AUTOSCALER_MIG_NAME}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
-    sed -i'' -e "s@{{kubemark_autoscaler_min_nodes}}@${KUBEMARK_AUTOSCALER_MIN_NODES}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
-    sed -i'' -e "s@{{kubemark_autoscaler_max_nodes}}@${KUBEMARK_AUTOSCALER_MAX_NODES}@g" "${RESOURCE_DIRECTORY}/addons/cluster-autoscaler.json"
-  fi
-
-  # Kube DNS.
-  if [[ "${ENABLE_KUBEMARK_KUBE_DNS:-}" == "true" ]]; then
-    echo "Setting up kube-dns"
-    sed "s@{{dns_domain}}@${KUBE_DNS_DOMAIN}@g" "${RESOURCE_DIRECTORY}/kube_dns_template.yaml" > "${RESOURCE_DIRECTORY}/addons/kube_dns.yaml"
-  fi
-
-  "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark"
-
-  # Create the replication controller for hollow-nodes.
-  # We allow to override the NUM_REPLICAS when running Cluster Autoscaler.
-  NUM_REPLICAS=${NUM_REPLICAS:-${NUM_NODES}}
-  sed "s@{{numreplicas}}@${NUM_REPLICAS}@g" "${RESOURCE_DIRECTORY}/hollow-node_template.yaml" > "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  proxy_cpu=20
-  if [ "${NUM_NODES}" -gt 1000 ]; then
-    proxy_cpu=50
-  fi
-  proxy_cpu=${KUBEMARK_HOLLOW_PROXY_MILLICPU:-$proxy_cpu}
-  proxy_mem_per_node=${KUBEMARK_HOLLOW_PROXY_MEM_PER_NODE_KB:-50}
-  proxy_mem=$((100 * 1024 + proxy_mem_per_node*NUM_NODES))
-  hollow_node_labels=${HOLLOW_NODE_LABELS:-$(calculate-node-labels)}
-  hollow_kubelet_params=$(eval "for param in ${HOLLOW_KUBELET_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
-  hollow_kubelet_params=${hollow_kubelet_params%?}
-  hollow_proxy_params=$(eval "for param in ${HOLLOW_PROXY_TEST_ARGS:-}; do echo -n \\\"\$param\\\",; done")
-  hollow_proxy_params=${hollow_proxy_params%?}
-
-  sed -i'' -e "s@{{hollow_kubelet_millicpu}}@${KUBEMARK_HOLLOW_KUBELET_MILLICPU:-40}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{hollow_kubelet_mem_Ki}}@${KUBEMARK_HOLLOW_KUBELET_MEM_KB:-$((100*1024))}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{hollow_proxy_millicpu}}@${proxy_cpu}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{hollow_proxy_mem_Ki}}@${proxy_mem}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{npd_millicpu}}@${KUBEMARK_NPD_MILLICPU:-20}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{npd_mem_Ki}}@${KUBEMARK_NPD_MEM_KB:-$((20*1024))}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{kubemark_image_registry}}@${KUBEMARK_IMAGE_REGISTRY}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{kubemark_image_tag}}@${KUBEMARK_IMAGE_TAG}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{master_ip}}@${MASTER_IP}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{hollow_node_labels}}@${hollow_node_labels}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{hollow_kubelet_params}}@${hollow_kubelet_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{hollow_proxy_params}}@${hollow_proxy_params}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  sed -i'' -e "s@{{kubemark_mig_config}}@${KUBEMARK_MIG_CONFIG:-}@g" "${RESOURCE_DIRECTORY}/hollow-node.yaml"
-  "${KUBECTL}" create -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark"
-
-  echo "Created secrets, configMaps, replication-controllers required for hollow-nodes."
-}
-
-# Wait until all hollow-nodes are running or there is a timeout.
-function wait-for-hollow-nodes-to-run-or-timeout {
-  echo -n "Waiting for all hollow-nodes to become Running"
-  start=$(date +%s)
-  nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true
-  ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1))
-
-  until [[ "${ready}" -ge "${NUM_REPLICAS}" ]]; do
-    echo -n "."
-    sleep 1
-    now=$(date +%s)
-    # Fail it if it already took more than 30 minutes.
-    if [ $((now - start)) -gt 1800 ]; then
-      echo ""
-      "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" describe node
-      # shellcheck disable=SC2154 # Color defined in sourced script
-      echo -e "${color_red} Timeout waiting for all hollow-nodes to become Running. ${color_norm}"
-      # Try listing nodes again - if it fails it means that API server is not responding
-      if "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node &> /dev/null; then
-        echo "Found only ${ready} ready hollow-nodes while waiting for ${NUM_NODES}."
-      else
-        echo "Got error while trying to list hollow-nodes. Probably API server is down."
-      fi
-      pods=$("${KUBECTL}" get pods -l name=hollow-node --namespace=kubemark) || true
-      running=$(($(echo "${pods}" | grep -c "Running")))
-      echo "${running} hollow-nodes are reported as 'Running'"
-      not_running=$(($(echo "${pods}" | grep -vc "Running") - 1))
-      echo "${not_running} hollow-nodes are reported as NOT 'Running'"
-      echo "${pods}" | grep -v Running
-      exit 1
-    fi
-    nodes=$("${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node 2> /dev/null) || true
-    ready=$(($(echo "${nodes}" | grep -vc "NotReady") - 1))
-  done
-  # shellcheck disable=SC2154 # Color defined in sourced script
-  echo -e "${color_green} Done!${color_norm}"
-}
-
-############################### Main Function ########################################
-
-# Setup for hollow-nodes.
-function start-hollow-nodes {
-  # shellcheck disable=SC2154 # Color defined in sourced script
-  echo -e "${color_yellow}STARTING SETUP FOR HOLLOW-NODES${color_norm}"
-  create-and-upload-hollow-node-image
-  create-kube-hollow-node-resources
-  wait-for-hollow-nodes-to-run-or-timeout
-}
-
-# Annotates the node objects in the kubemark cluster to make their size
-# similar to regular nodes.
-# TODO(#90833): Replace this with image preloading from ClusterLoader to better
-# reflect the reality in kubemark tests.
-function resize-node-objects {
-  if [[ -z "$KUBEMARK_NODE_OBJECT_SIZE_BYTES" ]]; then
-    return 0
-  fi
-
-  annotation_size_bytes="${KUBEMARK_NODE_OBJECT_SIZE_BYTES}"
-  echo "Annotating node objects with ${annotation_size_bytes} byte label"
-  label=$( (LC_CTYPE=C < /dev/urandom tr -dc 'a-zA-Z0-9' | fold -w "$annotation_size_bytes"; true) | head -n 1)
-  "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get nodes -o name \
-    | xargs -P50 -r -I% "${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" annotate --overwrite % label="$label"
-  echo "Annotating node objects completed"
-}
-
-
-detect-project &> /dev/null
-create-kubemark-master
-
-if [ -f "${INTERNAL_KUBECONFIG}" ]; then
-    HOLLOWNODE_KUBECONFIG="${INTERNAL_KUBECONFIG}"
-else
-    HOLLOWNODE_KUBECONFIG="${LOCAL_KUBECONFIG}"
-fi
-
-MASTER_IP=$(grep server "${HOLLOWNODE_KUBECONFIG}" | awk -F "/" '{print $3}')
-
-start-hollow-nodes
-resize-node-objects
-
-echo ""
-echo "Master IP: ${MASTER_IP}"
-echo "Kubeconfig for kubemark master is written in ${LOCAL_KUBECONFIG}"
@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2015 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Script that destroys Kubemark cluster and deletes all master resources.
-
-KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
-
-source "${KUBE_ROOT}/test/kubemark/skeleton/util.sh"
-source "${KUBE_ROOT}/test/kubemark/cloud-provider-config.sh"
-source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/util.sh"
-source "${KUBE_ROOT}/cluster/kubemark/${CLOUD_PROVIDER}/config-default.sh"
-
-if [[ -f "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/shutdown.sh" ]] ; then
-  source "${KUBE_ROOT}/test/kubemark/${CLOUD_PROVIDER}/shutdown.sh"
-fi
-
-source "${KUBE_ROOT}/cluster/kubemark/util.sh"
-
-KUBECTL="${KUBE_ROOT}/cluster/kubectl.sh"
-KUBEMARK_DIRECTORY="${KUBE_ROOT}/test/kubemark"
-RESOURCE_DIRECTORY="${KUBEMARK_DIRECTORY}/resources"
-
-detect-project &> /dev/null
-
-"${KUBECTL}" delete --timeout=5m -f "${RESOURCE_DIRECTORY}/addons" --namespace="kubemark" &> /dev/null || true
-"${KUBECTL}" delete --timeout=5m -f "${RESOURCE_DIRECTORY}/hollow-node.yaml" --namespace="kubemark" &> /dev/null || true
-"${KUBECTL}" delete --timeout=5m -f "${RESOURCE_DIRECTORY}/kubemark-ns.json" &> /dev/null || true
-
-rm -rf "${RESOURCE_DIRECTORY}/addons" \
-	"${RESOURCE_DIRECTORY}/kubeconfig.kubemark" \
-	"${RESOURCE_DIRECTORY}/hollow-node.yaml"  &> /dev/null || true
-
-delete-kubemark-master
				`@@ -1 +0,0 @@`
				`These resources are used to add extra (non-default) bindings to kubemark to match users and groups that are particular to the kubemark environment. These are not standard bootstrap bindings and not standard users they are bound to, and have been adapted from cluster/addons/e2e-rbac-bindings. Tighten/loosen these access rights as required in future.`