make it possible to mount ServiceAccount-mounted CA cert from ClusterTrustBundle

Signed-off-by: Stanislav Láznička <slznika@microsoft.com>

pkg/features/kube_features.go

@@ -556,6 +556,15 @@ const (
 	// Enables the fast path for inter-pod affinity calculations when the topology key is kubernetes.io/hostname.
 	InterPodAffinityHostnameFastPath featuregate.Feature = "InterPodAffinityHostnameFastPath"
 
+	// owner: @stlaz
+	// kep: https://kep.k8s.io/6283
+	//
+	// Makes the kube-apiserver use the ClusterTrustBundles with signer 'kubernetes.io/kube-apiserver-serving'
+	// when injecting the kube-apiserver serving certificate trust to workloads.
+	// This is injected instead of the 'kube-root-ca.crt' configMap into automounted ServiceAccount
+	// volumes.
+	KubeAPIServerWorkloadsTrust featuregate.Feature = "KubeAPIServerWorkloadsTrust"
+
 	// owner: @adrianmoisey @danwinship
 	// kep: https://kep.k8s.io/5495
 	//
@@ -1601,6 +1610,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 		{Version: version.MustParse("1.37"), Default: false, PreRelease: featuregate.Alpha},
 	},
 
+	KubeAPIServerWorkloadsTrust: {
+		{Version: version.MustParse("1.38"), Default: false, PreRelease: featuregate.Alpha},
+	},
+
 	KubeProxyIPVS: {
 		{Version: version.MustParse("1.0"), Default: true, PreRelease: featuregate.GA},
 		{Version: version.MustParse("1.37"), Default: true, PreRelease: featuregate.Deprecated},
@@ -2496,6 +2509,8 @@ var defaultKubernetesFeatureGateDependencies = map[featuregate.Feature][]feature
 
 	InterPodAffinityHostnameFastPath: {},
 
+	KubeAPIServerWorkloadsTrust: {ClusterTrustBundle, ClusterTrustBundleProjection},
+
 	KubeProxyIPVS: {},
 
 	KubeProxyNFTablesLocalhostNodePorts: {},

plugin/pkg/admission/serviceaccount/admission.go

@@ -32,11 +32,13 @@ import (
 	"k8s.io/apiserver/pkg/admission"
 	genericadmissioninitializer "k8s.io/apiserver/pkg/admission/initializer"
 	"k8s.io/apiserver/pkg/storage/names"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	corev1listers "k8s.io/client-go/listers/core/v1"
 	podutil "k8s.io/kubernetes/pkg/api/pod"
 	api "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/serviceaccount"
 	"k8s.io/utils/ptr"
 )
@@ -56,6 +58,10 @@ const (
 	// The token file would then be accessible at /var/run/secrets/kubernetes.io/serviceaccount
 	DefaultAPITokenMountPath = "/var/run/secrets/kubernetes.io/serviceaccount"
 
+	// KubeAPIServerServingSigner is the signer that is used for distribution of the
+	// kube-apiserver serving trust
+	KubeAPIServerServingSigner = "kubernetes.io/kube-apiserver-serving"
+
 	// PluginName is the name of this admission plugin
 	PluginName = "ServiceAccount"
 )
@@ -485,6 +491,29 @@ func (s *Plugin) mountServiceAccountToken(serviceAccount *corev1.ServiceAccount,
 
 // TokenVolumeSource returns the projected volume source for service account token.
 func TokenVolumeSource() *api.ProjectedVolumeSource {
+	kubeAPIServerTrustProjection := api.VolumeProjection{
+		ConfigMap: &api.ConfigMapProjection{
+			LocalObjectReference: api.LocalObjectReference{
+				Name: "kube-root-ca.crt",
+			},
+			Items: []api.KeyToPath{
+				{
+					Key:  "ca.crt",
+					Path: "ca.crt",
+				},
+			},
+		},
+	}
+	if utilfeature.DefaultFeatureGate.Enabled(features.KubeAPIServerWorkloadsTrust) {
+		kubeAPIServerTrustProjection = api.VolumeProjection{
+			ClusterTrustBundle: &api.ClusterTrustBundleProjection{
+				SignerName:    new(KubeAPIServerServingSigner),
+				LabelSelector: &metav1.LabelSelector{},
+				Path:          "ca.crt",
+			},
+		}
+	}
+
 	return &api.ProjectedVolumeSource{
 		// explicitly set default value, see #104464
 		DefaultMode: ptr.To[int32](corev1.ProjectedVolumeSourceDefaultMode),
@@ -495,19 +524,7 @@ func TokenVolumeSource() *api.ProjectedVolumeSource {
 					ExpirationSeconds: ptr.To[int64](serviceaccount.WarnOnlyBoundTokenExpirationSeconds),
 				},
 			},
-			{
-				ConfigMap: &api.ConfigMapProjection{
-					LocalObjectReference: api.LocalObjectReference{
-						Name: "kube-root-ca.crt",
-					},
-					Items: []api.KeyToPath{
-						{
-							Key:  "ca.crt",
-							Path: "ca.crt",
-						},
-					},
-				},
-			},
+			kubeAPIServerTrustProjection,
 			{
 				DownwardAPI: &api.DownwardAPIProjection{
 					Items: []api.DownwardAPIVolumeFile{

plugin/pkg/admission/serviceaccount/admission_test.go

@@ -18,6 +18,7 @@ package serviceaccount
 
 import (
 	"context"
+	"fmt"
 	"reflect"
 	"strings"
 	"testing"
@@ -29,11 +30,14 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apiserver/pkg/admission"
 	admissiontesting "k8s.io/apiserver/pkg/admission/testing"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes/fake"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
 	api "k8s.io/kubernetes/pkg/apis/core"
 	v1defaults "k8s.io/kubernetes/pkg/apis/core/v1"
 	"k8s.io/kubernetes/pkg/controller"
+	"k8s.io/kubernetes/pkg/features"
 	kubelet "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/utils/ptr"
 )
@@ -162,36 +166,7 @@ func TestRejectsMirrorPodWithServiceAccountTokenVolumeProjections(t *testing.T)
 func TestAssignsDefaultServiceAccountAndBoundTokenWithNoSecretTokens(t *testing.T) {
 	ns := "myns"
 
-	admit := NewServiceAccount()
-	informerFactory := informers.NewSharedInformerFactory(nil, controller.NoResyncPeriodFunc())
-	admit.SetExternalKubeInformerFactory(informerFactory)
-	admit.MountServiceAccountToken = true
-
-	// Add the default service account for the ns into the cache
-	informerFactory.Core().V1().ServiceAccounts().Informer().GetStore().Add(&corev1.ServiceAccount{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      DefaultServiceAccountName,
-			Namespace: ns,
-		},
-	})
-
-	v1PodIn := &corev1.Pod{
-		Spec: corev1.PodSpec{
-			Containers: []corev1.Container{{}},
-		},
-	}
-	v1defaults.SetObjectDefaults_Pod(v1PodIn)
-	pod := &api.Pod{}
-	if err := v1defaults.Convert_v1_Pod_To_core_Pod(v1PodIn, pod, nil); err != nil {
-		t.Fatal(err)
-	}
-	attrs := admission.NewAttributesRecord(pod, nil, api.Kind("Pod").WithVersion("version"), ns, "myname", api.Resource("pods").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil)
-	err := admissiontesting.WithReinvocationTesting(t, admit).Admit(context.TODO(), attrs, nil)
-	if err != nil {
-		t.Fatalf("Expected success, got: %v", err)
-	}
-
-	expectedVolumes := []api.Volume{{
+	expectedConfigMapVolumes := []api.Volume{{
 		Name: "cleared",
 		VolumeSource: api.VolumeSource{
 			Projected: &api.ProjectedVolumeSource{
@@ -204,40 +179,95 @@ func TestAssignsDefaultServiceAccountAndBoundTokenWithNoSecretTokens(t *testing.
 			},
 		},
 	}}
-	expectedVolumeMounts := []api.VolumeMount{{
-		Name:      "cleared",
-		ReadOnly:  true,
-		MountPath: "/var/run/secrets/kubernetes.io/serviceaccount",
+	expectedVolumesClusterTrustBundle := []api.Volume{{
+		Name: "cleared",
+		VolumeSource: api.VolumeSource{
+			Projected: &api.ProjectedVolumeSource{
+				Sources: []api.VolumeProjection{
+					{ServiceAccountToken: &api.ServiceAccountTokenProjection{ExpirationSeconds: ptr.To[int64](3607), Path: "token"}},
+					{ClusterTrustBundle: &api.ClusterTrustBundleProjection{SignerName: new("kubernetes.io/kube-apiserver-serving"), LabelSelector: &metav1.LabelSelector{}, Path: "ca.crt"}},
+					{DownwardAPI: &api.DownwardAPIProjection{Items: []api.DownwardAPIVolumeFile{{Path: "namespace", FieldRef: &api.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}},
+				},
+				DefaultMode: ptr.To[int32](0644),
+			},
+		},
 	}}
 
-	// clear generated volume names
-	for i := range pod.Spec.Volumes {
-		if len(pod.Spec.Volumes[i].Name) > 0 {
-			pod.Spec.Volumes[i].Name = "cleared"
+	for _, ctbTrustEnabled := range []bool{false, true} {
+		expectedVolumes := expectedConfigMapVolumes
+		if ctbTrustEnabled {
+			expectedVolumes = expectedVolumesClusterTrustBundle
 		}
-	}
-	for i := range pod.Spec.Containers[0].VolumeMounts {
-		if len(pod.Spec.Containers[0].VolumeMounts[i].Name) > 0 {
-			pod.Spec.Containers[0].VolumeMounts[i].Name = "cleared"
-		}
-	}
 
-	if !reflect.DeepEqual(expectedVolumes, pod.Spec.Volumes) {
-		t.Errorf("unexpected volumes: %s", cmp.Diff(expectedVolumes, pod.Spec.Volumes))
-	}
-	if !reflect.DeepEqual(expectedVolumeMounts, pod.Spec.Containers[0].VolumeMounts) {
-		t.Errorf("unexpected volumes: %s", cmp.Diff(expectedVolumeMounts, pod.Spec.Containers[0].VolumeMounts))
-	}
+		t.Run(fmt.Sprintf("KubeAPIServerWorkloadsTrust=%t", ctbTrustEnabled), func(t *testing.T) {
+			featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeAPIServerWorkloadsTrust, ctbTrustEnabled)
 
-	// ensure result converted to v1 matches defaulted object
-	v1PodOut := &corev1.Pod{}
-	if err := v1defaults.Convert_core_Pod_To_v1_Pod(pod, v1PodOut, nil); err != nil {
-		t.Fatal(err)
-	}
-	v1PodOutDefaulted := v1PodOut.DeepCopy()
-	v1defaults.SetObjectDefaults_Pod(v1PodOutDefaulted)
-	if !reflect.DeepEqual(v1PodOut, v1PodOutDefaulted) {
-		t.Error(cmp.Diff(v1PodOut, v1PodOutDefaulted))
+			admit := NewServiceAccount()
+			informerFactory := informers.NewSharedInformerFactory(nil, controller.NoResyncPeriodFunc())
+			admit.SetExternalKubeInformerFactory(informerFactory)
+			admit.MountServiceAccountToken = true
+
+			// Add the default service account for the ns into the cache
+			err := informerFactory.Core().V1().ServiceAccounts().Informer().GetStore().Add(&corev1.ServiceAccount{
+				Name:      DefaultServiceAccountName,
+				Namespace: ns,
+			})
+			if err != nil {
+				t.Fatalf("failed to add ServiceAccount to the informer's store: %v", err)
+			}
+
+			v1PodIn := &corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{{}},
+				},
+			}
+			v1defaults.SetObjectDefaults_Pod(v1PodIn)
+			pod := &api.Pod{}
+			if err := v1defaults.Convert_v1_Pod_To_core_Pod(v1PodIn, pod, nil); err != nil {
+				t.Fatal(err)
+			}
+			attrs := admission.NewAttributesRecord(pod, nil, api.Kind("Pod").WithVersion("version"), ns, "myname", api.Resource("pods").WithVersion("version"), "", admission.Create, &metav1.CreateOptions{}, false, nil)
+			err = admissiontesting.WithReinvocationTesting(t, admit).Admit(context.TODO(), attrs, nil)
+			if err != nil {
+				t.Fatalf("Expected success, got: %v", err)
+			}
+
+			expectedVolumeMounts := []api.VolumeMount{{
+				Name:      "cleared",
+				ReadOnly:  true,
+				MountPath: "/var/run/secrets/kubernetes.io/serviceaccount",
+			}}
+
+			// clear generated volume names
+			for i := range pod.Spec.Volumes {
+				if len(pod.Spec.Volumes[i].Name) > 0 {
+					pod.Spec.Volumes[i].Name = "cleared"
+				}
+			}
+			for i := range pod.Spec.Containers[0].VolumeMounts {
+				if len(pod.Spec.Containers[0].VolumeMounts[i].Name) > 0 {
+					pod.Spec.Containers[0].VolumeMounts[i].Name = "cleared"
+				}
+			}
+
+			if !reflect.DeepEqual(expectedVolumes, pod.Spec.Volumes) {
+				t.Errorf("unexpected volumes: %s", cmp.Diff(expectedVolumes, pod.Spec.Volumes))
+			}
+			if !reflect.DeepEqual(expectedVolumeMounts, pod.Spec.Containers[0].VolumeMounts) {
+				t.Errorf("unexpected volumes: %s", cmp.Diff(expectedVolumeMounts, pod.Spec.Containers[0].VolumeMounts))
+			}
+
+			// ensure result converted to v1 matches defaulted object
+			v1PodOut := &corev1.Pod{}
+			if err := v1defaults.Convert_core_Pod_To_v1_Pod(pod, v1PodOut, nil); err != nil {
+				t.Fatal(err)
+			}
+			v1PodOutDefaulted := v1PodOut.DeepCopy()
+			v1defaults.SetObjectDefaults_Pod(v1PodOutDefaulted)
+			if !reflect.DeepEqual(v1PodOut, v1PodOutDefaulted) {
+				t.Error(cmp.Diff(v1PodOut, v1PodOutDefaulted))
+			}
+		})
 	}
 }
 
@@ -288,6 +318,12 @@ func TestDeniesInvalidServiceAccount(t *testing.T) {
 }
 
 func TestAutomountsAPIToken(t *testing.T) {
+	for _, ctbTrustEnabled := range []bool{false, true} {
+		t.Run(fmt.Sprintf("KubeAPIServerWorkloadsTrust=%t", ctbTrustEnabled), func(t *testing.T) { testAutomountsAPIToken(t, ctbTrustEnabled) })
+	}
+}
+func testAutomountsAPIToken(t *testing.T, ctbTrustEnabled bool) {
+	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeAPIServerWorkloadsTrust, ctbTrustEnabled)
 
 	admit := NewServiceAccount()
 	informerFactory := informers.NewSharedInformerFactory(nil, controller.NoResyncPeriodFunc())

test/compatibility_lifecycle/reference/feature_list.md

@@ -115,6 +115,7 @@
 | InformerResourceVersion | :ballot_box_with_check: 1.35+ |  | 1.30–1.34 |  | 1.35– |  |  | [code](https://cs.k8s.io/?q=%5CbInformerResourceVersion%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbInformerResourceVersion%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
 | InterPodAffinityHostnameFastPath |  |  | 1.37– |  |  |  |  | [code](https://cs.k8s.io/?q=%5CbInterPodAffinityHostnameFastPath%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbInterPodAffinityHostnameFastPath%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
 | KMSv1 | :ballot_box_with_check: 1.0+ |  |  |  | 1.0–1.27 | 1.28– |  | [code](https://cs.k8s.io/?q=%5CbKMSv1%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbKMSv1%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
+| KubeAPIServerWorkloadsTrust |  |  | 1.38– |  |  |  | ClusterTrustBundle<br>ClusterTrustBundleProjection | [code](https://cs.k8s.io/?q=%5CbKubeAPIServerWorkloadsTrust%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbKubeAPIServerWorkloadsTrust%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
 | KubeProxyIPVS | :ballot_box_with_check:&nbsp;1.0+ |  |  |  | 1.0–1.36 | 1.37– |  | [code](https://cs.k8s.io/?q=%5CbKubeProxyIPVS%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbKubeProxyIPVS%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
 | KubeProxyNFTablesLocalhostNodePorts | :ballot_box_with_check:&nbsp;1.38+ |  | 1.37 | 1.38– |  |  |  | [code](https://cs.k8s.io/?q=%5CbKubeProxyNFTablesLocalhostNodePorts%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbKubeProxyNFTablesLocalhostNodePorts%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |
 | KubeletAllocatedPodsEndpoint |  |  | 1.37– |  |  |  | InPlacePodVerticalScaling | [code](https://cs.k8s.io/?q=%5CbKubeletAllocatedPodsEndpoint%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbKubeletAllocatedPodsEndpoint%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) |

test/compatibility_lifecycle/reference/versioned_feature_list.yaml

@@ -953,6 +953,12 @@
     lockToDefault: false
     preRelease: Deprecated
     version: "1.29"
+- name: KubeAPIServerWorkloadsTrust
+  versionedSpecs:
+  - default: false
+    lockToDefault: false
+    preRelease: Alpha
+    version: "1.38"
 - name: KubeletAllocatedPodsEndpoint
   versionedSpecs:
   - default: false

test/e2e/auth/service_accounts.go

@@ -54,7 +54,10 @@ import (
 	"github.com/onsi/gomega"
 )
 
-const rootCAConfigMapName = "kube-root-ca.crt"
+const (
+	rootCAConfigMapName        = "kube-root-ca.crt"
+	kubeAPIServerServingSigner = "kubernetes.io/kube-apiserver-serving"
+)
 
 var _ = SIGDescribe("ServiceAccounts", func() {
 	f := framework.NewDefaultFramework("svcaccounts")
@@ -166,6 +169,37 @@ var _ = SIGDescribe("ServiceAccounts", func() {
 		}
 	})
 
+	framework.It("should mount CA from signerName='kubernetes.io/kube-apiserver-serving' ClusterTrustBundle into pods", f.WithFeatureGate(features.KubeAPIServerWorkloadsTrust), func(ctx context.Context) {
+		sa, err := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).Create(ctx, &v1.ServiceAccount{Name: "mount-test"}, metav1.CreateOptions{})
+		framework.ExpectNoError(err)
+
+		pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, &v1.Pod{
+			Name: "pod-service-account-" + string(uuid.NewUUID()),
+			Spec: v1.PodSpec{
+				ServiceAccountName: sa.Name,
+				Containers: []v1.Container{{
+					Name:    "test",
+					Image:   imageutils.GetE2EImage(imageutils.BusyBox),
+					Command: []string{"sleep", "100000"},
+				}},
+				TerminationGracePeriodSeconds: new(int64(0)),
+				RestartPolicy:                 v1.RestartPolicyNever,
+			},
+		}, metav1.CreateOptions{})
+		framework.ExpectNoError(err)
+		framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod))
+
+		tk := e2ekubectl.NewTestKubeconfig(framework.TestContext.CertDir, framework.TestContext.Host, framework.TestContext.KubeConfig, framework.TestContext.KubeContext, framework.TestContext.KubectlPath, f.Namespace.Name)
+		mountedCA, err := tk.ReadFileViaContainer(pod.Name, pod.Spec.Containers[0].Name, path.Join(serviceaccount.DefaultAPITokenMountPath, v1.ServiceAccountRootCAKey))
+		framework.ExpectNoError(err)
+
+		servingTrustBundles, err := f.ClientSet.CertificatesV1().ClusterTrustBundles().List(ctx, metav1.ListOptions{FieldSelector: fmt.Sprintf("spec.signerName=%s", kubeAPIServerServingSigner)})
+		framework.ExpectNoError(err)
+		gomega.Expect(servingTrustBundles.Items).To(gomega.HaveLen(1)) // currently we only ever create a single ClusterTrustBundle for the signer
+
+		gomega.Expect(mountedCA).To(gomega.Equal(servingTrustBundles.Items[0].Spec.TrustBundle))
+	})
+
 	/*
 	   Release: v1.9
 	   Testname: Service account tokens auto mount optionally