diff --git a/pkg/kubeapiserver/authorizer/config.go b/pkg/kubeapiserver/authorizer/config.go index 7a778ebd867..f765141b334 100644 --- a/pkg/kubeapiserver/authorizer/config.go +++ b/pkg/kubeapiserver/authorizer/config.go @@ -68,6 +68,9 @@ type Config struct { // AuthorizationConfiguration stores the configuration for the Authorizer chain // It will deprecate most of the above flags when GA AuthorizationConfiguration *authzconfig.AuthorizationConfiguration + // InitialAuthorizationConfigurationData holds the initial authorization configuration data + // that was read from the authorization configuration file. + InitialAuthorizationConfigurationData string } // New returns the right sort of union of multiple authorizer.Authorizer objects @@ -84,6 +87,7 @@ func (config Config) New(ctx context.Context, serverID string) (authorizer.Autho initialConfig: config, apiServerID: serverID, lastLoadedConfig: config.AuthorizationConfiguration, + lastReadData: []byte(config.InitialAuthorizationConfigurationData), reloadInterval: time.Minute, compiler: authorizationcel.NewDefaultCompiler(), } @@ -160,12 +164,16 @@ func GetNameForAuthorizerMode(mode string) string { return strings.ToLower(mode) } -func LoadAndValidateFile(configFile string, compiler authorizationcel.Compiler, requireNonWebhookTypes sets.Set[authzconfig.AuthorizerType]) (*authzconfig.AuthorizationConfiguration, error) { +func LoadAndValidateFile(configFile string, compiler authorizationcel.Compiler, requireNonWebhookTypes sets.Set[authzconfig.AuthorizerType]) (*authzconfig.AuthorizationConfiguration, string, error) { data, err := os.ReadFile(configFile) if err != nil { - return nil, err + return nil, "", err } - return LoadAndValidateData(data, compiler, requireNonWebhookTypes) + config, err := LoadAndValidateData(data, compiler, requireNonWebhookTypes) + if err != nil { + return nil, "", err + } + return config, string(data), nil } func LoadAndValidateData(data []byte, compiler authorizationcel.Compiler, requireNonWebhookTypes sets.Set[authzconfig.AuthorizerType]) (*authzconfig.AuthorizationConfiguration, error) { @@ -198,7 +206,6 @@ func LoadAndValidateData(data []byte, compiler authorizationcel.Compiler, requir if expectedName != authorizer.Name { allErrors = append(allErrors, fmt.Errorf("expected name %s for authorizer %s instead of %s", expectedName, authorizer.Type, authorizer.Name)) } - } if missingTypes := requireNonWebhookTypes.Difference(seenModes); missingTypes.Len() > 0 { diff --git a/pkg/kubeapiserver/authorizer/reload.go b/pkg/kubeapiserver/authorizer/reload.go index e6995631f6d..e559153bfa4 100644 --- a/pkg/kubeapiserver/authorizer/reload.go +++ b/pkg/kubeapiserver/authorizer/reload.go @@ -192,7 +192,7 @@ type kubeapiserverWebhookMetrics struct { // Blocks until ctx is complete. func (r *reloadableAuthorizerResolver) runReload(ctx context.Context) { metrics.RegisterMetrics() - metrics.RecordAuthorizationConfigAutomaticReloadSuccess(r.apiServerID) + metrics.RecordAuthorizationConfigLastConfigInfo(r.apiServerID, string(r.lastReadData)) filesystem.WatchUntil( ctx, @@ -250,5 +250,5 @@ func (r *reloadableAuthorizerResolver) checkFile(ctx context.Context) { ruleResolver: ruleResolver, }) klog.InfoS("reloaded authz config") - metrics.RecordAuthorizationConfigAutomaticReloadSuccess(r.apiServerID) + metrics.RecordAuthorizationConfigAutomaticReloadSuccess(r.apiServerID, string(data)) } diff --git a/pkg/kubeapiserver/options/authentication.go b/pkg/kubeapiserver/options/authentication.go index 5fc4842d324..4f9ebeae0eb 100644 --- a/pkg/kubeapiserver/options/authentication.go +++ b/pkg/kubeapiserver/options/authentication.go @@ -727,6 +727,7 @@ func (o *BuiltInAuthenticationOptions) ApplyTo( if len(o.AuthenticationConfigFile) > 0 { authenticationconfigmetrics.RegisterMetrics() + authenticationconfigmetrics.RecordAuthenticationConfigLastConfigInfo(apiServerID, authenticatorConfig.AuthenticationConfigData) trackedAuthenticationConfigData := authenticatorConfig.AuthenticationConfigData var mu sync.Mutex @@ -789,7 +790,7 @@ func (o *BuiltInAuthenticationOptions) ApplyTo( trackedAuthenticationConfigData = authConfigData klog.InfoS("reloaded authentication config") - authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID) + authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID, authConfigData) }, func(err error) { klog.ErrorS(err, "watching authentication config file") }, ) diff --git a/pkg/kubeapiserver/options/authorization.go b/pkg/kubeapiserver/options/authorization.go index 8b36309c709..2edfa11309d 100644 --- a/pkg/kubeapiserver/options/authorization.go +++ b/pkg/kubeapiserver/options/authorization.go @@ -118,7 +118,7 @@ func (o *BuiltInAuthorizationOptions) Validate() []error { } // load/validate kube-apiserver authz config with no opinion about required modes - _, err := authorizer.LoadAndValidateFile(o.AuthorizationConfigurationFile, authorizationcel.NewDefaultCompiler(), nil) + _, _, err := authorizer.LoadAndValidateFile(o.AuthorizationConfigurationFile, authorizationcel.NewDefaultCompiler(), nil) if err != nil { return append(allErrors, err) } @@ -219,6 +219,7 @@ func (o *BuiltInAuthorizationOptions) ToAuthorizationConfig(versionedInformerFac var authorizationConfiguration *authzconfig.AuthorizationConfiguration var err error + var authorizationConfigData string // if --authorization-config is set, check if // - the feature flag is set @@ -236,7 +237,7 @@ func (o *BuiltInAuthorizationOptions) ToAuthorizationConfig(versionedInformerFac return nil, fmt.Errorf("--%s can not be specified when --%s or --authorization-webhook-* flags are defined", authorizationConfigFlag, authorizationModeFlag) } // load/validate kube-apiserver authz config with no opinion about required modes - authorizationConfiguration, err = authorizer.LoadAndValidateFile(o.AuthorizationConfigurationFile, authorizationcel.NewDefaultCompiler(), nil) + authorizationConfiguration, authorizationConfigData, err = authorizer.LoadAndValidateFile(o.AuthorizationConfigurationFile, authorizationcel.NewDefaultCompiler(), nil) if err != nil { return nil, err } @@ -252,8 +253,9 @@ func (o *BuiltInAuthorizationOptions) ToAuthorizationConfig(versionedInformerFac VersionedInformerFactory: versionedInformerFactory, WebhookRetryBackoff: o.WebhookRetryBackoff, - ReloadFile: o.AuthorizationConfigurationFile, - AuthorizationConfiguration: authorizationConfiguration, + ReloadFile: o.AuthorizationConfigurationFile, + AuthorizationConfiguration: authorizationConfiguration, + InitialAuthorizationConfigurationData: authorizationConfigData, }, nil } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authenticationconfig/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/server/options/authenticationconfig/metrics/metrics.go index 74925972ba1..ed878d94efe 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authenticationconfig/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authenticationconfig/metrics/metrics.go @@ -21,6 +21,7 @@ import ( "fmt" "sync" + "k8s.io/apiserver/pkg/util/configmetrics" "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" ) @@ -52,14 +53,25 @@ var ( }, []string{"status", "apiserver_id_hash"}, ) + + authenticationConfigLastConfigInfo = metrics.NewDesc( + metrics.BuildFQName(namespace, subsystem, "last_config_info"), + "Information about the last applied authentication configuration with hash as label, split by apiserver identity.", + []string{"apiserver_id_hash", "hash"}, + nil, + metrics.ALPHA, + "", + ) ) var registerMetrics sync.Once +var configHashProvider = configmetrics.NewAtomicHashProvider() func RegisterMetrics() { registerMetrics.Do(func() { legacyregistry.MustRegister(authenticationConfigAutomaticReloadsTotal) legacyregistry.MustRegister(authenticationConfigAutomaticReloadLastTimestampSeconds) + legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(authenticationConfigLastConfigInfo, configHashProvider)) }) } @@ -74,10 +86,16 @@ func RecordAuthenticationConfigAutomaticReloadFailure(apiServerID string) { authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime() } -func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID string) { +func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID, authConfigData string) { apiServerIDHash := getHash(apiServerID) authenticationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc() authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime() + + RecordAuthenticationConfigLastConfigInfo(apiServerID, authConfigData) +} + +func RecordAuthenticationConfigLastConfigInfo(apiServerID, authConfigData string) { + configHashProvider.SetHashes(getHash(apiServerID), getHash(authConfigData)) } func getHash(data string) string { diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authenticationconfig/metrics/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/authenticationconfig/metrics/metrics_test.go index 1722388855f..4af9f644630 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authenticationconfig/metrics/metrics_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authenticationconfig/metrics/metrics_test.go @@ -27,6 +27,18 @@ import ( const ( testAPIServerID = "testAPIServerID" testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37" + testConfigData = ` +apiVersion: apiserver.config.k8s.io/v1 +kind: AuthenticationConfiguration +jwt: +- issuer: + url: https://test-issuer + audiences: [ "aud" ] + claimMappings: + username: + claim: sub + prefix: "" +` ) func TestRecordAuthenticationConfigAutomaticReloadFailure(t *testing.T) { @@ -53,15 +65,19 @@ func TestRecordAuthenticationConfigAutomaticReloadSuccess(t *testing.T) { # HELP apiserver_authentication_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authentication configuration split by status and apiserver identity. # TYPE apiserver_authentication_config_controller_automatic_reloads_total counter apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1 + # HELP apiserver_authentication_config_controller_last_config_info [ALPHA] Information about the last applied authentication configuration with hash as label, split by apiserver identity. + # TYPE apiserver_authentication_config_controller_last_config_info gauge + apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:ccbcaf98557c273dfc779222d54a5bd3e785ea5330048f3bf4278cf3997b669c"} 1 ` metrics := []string{ namespace + "_" + subsystem + "_automatic_reloads_total", + namespace + "_" + subsystem + "_last_config_info", } - authenticationConfigAutomaticReloadsTotal.Reset() + ResetMetricsForTest() RegisterMetrics() - RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID) + RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, testConfigData) if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { t.Fatal(err) } @@ -107,3 +123,34 @@ func TestAuthenticationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) { } } } + +func TestRecordAuthenticationConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) { + ResetMetricsForTest() + RegisterMetrics() + + // Record initial success with first config + firstConfig := "config1" + RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, firstConfig) + + // Record success with different config - should clean up old metric + secondConfig := "config2" + RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, secondConfig) + + // Verify only the latest hash is present + expectedValue := ` + # HELP apiserver_authentication_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authentication configuration split by status and apiserver identity. + # TYPE apiserver_authentication_config_controller_automatic_reloads_total counter + apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 2 + # HELP apiserver_authentication_config_controller_last_config_info [ALPHA] Information about the last applied authentication configuration with hash as label, split by apiserver identity. + # TYPE apiserver_authentication_config_controller_last_config_info gauge + apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1 + ` + metrics := []string{ + namespace + "_" + subsystem + "_automatic_reloads_total", + namespace + "_" + subsystem + "_last_config_info", + } + + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { + t.Fatal(err) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics/metrics.go index e03894cce0a..1a9d1a09eff 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics/metrics.go @@ -22,6 +22,7 @@ import ( "hash" "sync" + "k8s.io/apiserver/pkg/util/configmetrics" "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" ) @@ -53,10 +54,20 @@ var ( }, []string{"status", "apiserver_id_hash"}, ) + + authorizationConfigLastConfigInfo = metrics.NewDesc( + metrics.BuildFQName(namespace, subsystem, "last_config_info"), + "Information about the last applied authorization configuration with hash as label, split by apiserver identity.", + []string{"apiserver_id_hash", "hash"}, + nil, + metrics.ALPHA, + "", + ) ) var registerMetrics sync.Once var hashPool *sync.Pool +var configHashProvider = configmetrics.NewAtomicHashProvider() func RegisterMetrics() { registerMetrics.Do(func() { @@ -67,6 +78,7 @@ func RegisterMetrics() { } legacyregistry.MustRegister(authorizationConfigAutomaticReloadsTotal) legacyregistry.MustRegister(authorizationConfigAutomaticReloadLastTimestampSeconds) + legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(authorizationConfigLastConfigInfo, configHashProvider)) }) } @@ -81,10 +93,16 @@ func RecordAuthorizationConfigAutomaticReloadFailure(apiServerID string) { authorizationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime() } -func RecordAuthorizationConfigAutomaticReloadSuccess(apiServerID string) { +func RecordAuthorizationConfigAutomaticReloadSuccess(apiServerID, authzConfigData string) { apiServerIDHash := getHash(apiServerID) authorizationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc() authorizationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime() + + RecordAuthorizationConfigLastConfigInfo(apiServerID, authzConfigData) +} + +func RecordAuthorizationConfigLastConfigInfo(apiServerID, authzConfigData string) { + configHashProvider.SetHashes(getHash(apiServerID), getHash(authzConfigData)) } func getHash(data string) string { diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics/metrics_test.go index 546561b289f..21c52dc0010 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics/metrics_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/authorizationconfig/metrics/metrics_test.go @@ -27,6 +27,27 @@ import ( const ( testAPIServerID = "testAPIServerID" testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37" + testConfigData = ` +apiVersion: apiserver.config.k8s.io/v1 +kind: AuthorizationConfiguration +authorizers: +- type: Webhook + name: error.example.com + webhook: + timeout: 5s + failurePolicy: Deny + subjectAccessReviewVersion: v1 + matchConditionSubjectAccessReviewVersion: v1 + authorizedTTL: 1ms + unauthorizedTTL: 1ms + connectionInfo: + type: KubeConfigFile + kubeConfigFile: /path/to/kubeconfig + matchConditions: + - expression: has(request.resourceAttributes) + - expression: 'request.resourceAttributes.namespace == "fail"' + - expression: 'request.resourceAttributes.name == "error"' +` ) func TestRecordAuthorizationConfigAutomaticReloadFailure(t *testing.T) { @@ -53,15 +74,19 @@ func TestRecordAuthorizationConfigAutomaticReloadSuccess(t *testing.T) { # HELP apiserver_authorization_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authorization configuration split by status and apiserver identity. # TYPE apiserver_authorization_config_controller_automatic_reloads_total counter apiserver_authorization_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1 + # HELP apiserver_authorization_config_controller_last_config_info [ALPHA] Information about the last applied authorization configuration with hash as label, split by apiserver identity. + # TYPE apiserver_authorization_config_controller_last_config_info gauge + apiserver_authorization_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:2a66c95e5593fc74e6c3dbbb708741361f0690ed45d17e4d4ac0b9c282b6538f"} 1 ` metrics := []string{ namespace + "_" + subsystem + "_automatic_reloads_total", + namespace + "_" + subsystem + "_last_config_info", } - authorizationConfigAutomaticReloadsTotal.Reset() + ResetMetricsForTest() RegisterMetrics() - RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID) + RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, testConfigData) if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { t.Fatal(err) } @@ -107,3 +132,34 @@ func TestAuthorizationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) { } } } + +func TestRecordAuthorizationConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) { + ResetMetricsForTest() + RegisterMetrics() + + // Record initial success with first config + firstConfig := "config1" + RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, firstConfig) + + // Record success with different config - should clean up old metric + secondConfig := "config2" + RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, secondConfig) + + // Verify only the latest hash is present + expectedValue := ` + # HELP apiserver_authorization_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authorization configuration split by status and apiserver identity. + # TYPE apiserver_authorization_config_controller_automatic_reloads_total counter + apiserver_authorization_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 2 + # HELP apiserver_authorization_config_controller_last_config_info [ALPHA] Information about the last applied authorization configuration with hash as label, split by apiserver identity. + # TYPE apiserver_authorization_config_controller_last_config_info gauge + apiserver_authorization_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1 + ` + metrics := []string{ + namespace + "_" + subsystem + "_automatic_reloads_total", + namespace + "_" + subsystem + "_last_config_info", + } + + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil { + t.Fatal(err) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go index 48efe7a2779..6d0e28428e7 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go @@ -907,9 +907,8 @@ func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte, // computeEncryptionConfigHash returns the expected hash for an encryption config file that has been loaded as bytes. // We use a hash instead of the raw file contents when tracking changes to avoid holding any encryption keys in memory outside of their associated transformers. -// This hash must be used in-memory and not externalized to the process because it has no cross-release stability guarantees. func computeEncryptionConfigHash(data []byte) string { - return fmt.Sprintf("k8s:enc:unstable:1:%x", sha256.Sum256(data)) + return fmt.Sprintf("sha256:%x", sha256.Sum256(data)) } var _ storagevalue.ResourceTransformers = &DynamicTransformers{} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go index 5e551e0962f..125d43ef93e 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go @@ -1843,7 +1843,7 @@ func errString(err error) string { func TestComputeEncryptionConfigHash(t *testing.T) { // hash the empty string to be sure that sha256 is being used - expect := "k8s:enc:unstable:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + expect := "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" sum := computeEncryptionConfigHash([]byte("")) if expect != sum { t.Errorf("expected hash %q but got %q", expect, sum) @@ -2220,7 +2220,7 @@ func TestGetEncryptionConfigHash(t *testing.T) { { name: "valid file", filepath: "testdata/valid-configs/secret-box-first.yaml", - wantHash: "k8s:enc:unstable:1:c638c0327dbc3276dd1fcf3e67895d19ebca16b91ae0d19af24ef0759b8e0f66", + wantHash: "sha256:c638c0327dbc3276dd1fcf3e67895d19ebca16b91ae0d19af24ef0759b8e0f66", wantErr: ``, }, } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go index fd783f41a44..5c249bcc08b 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go @@ -174,7 +174,7 @@ func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Conte } if updatedEffectiveConfig && err == nil { - metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID) + metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID, encryptionConfiguration.EncryptionFileContentHash) } if err != nil { diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller_test.go index 90074632419..50eca7b93d2 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller_test.go @@ -47,6 +47,9 @@ func TestController(t *testing.T) { # HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity. # TYPE apiserver_encryption_config_controller_automatic_reloads_total counter apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",status="success"} 1 +# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity. +# TYPE apiserver_encryption_config_controller_last_config_info gauge +apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",hash="sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef"} 1 ` const expectedFailureMetricValue = ` # HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity. @@ -67,7 +70,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }{ { name: "when invalid config is provided previous config shouldn't be changed", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 1, wantHashCalls: 1, wantTransformerClosed: true, @@ -82,7 +85,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, { name: "when new valid config is provided it should be updated", - wantECFileHash: "some new config hash", + wantECFileHash: "sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef", // some new config hash wantLoadCalls: 1, wantHashCalls: 1, wantMetrics: expectedSuccessMetricValue, @@ -98,13 +101,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash err: nil, }, }, - EncryptionFileContentHash: "some new config hash", + EncryptionFileContentHash: "sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef", // some new config hash }, nil }, }, { name: "when same valid config is provided previous config shouldn't be changed", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 1, wantHashCalls: 1, wantTransformerClosed: true, @@ -122,13 +125,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, }, // hash of initial "testdata/ec_config.yaml" config file before reloading - EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + EncryptionFileContentHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", }, nil }, }, { name: "when transformer's health check fails previous config shouldn't be changed", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 1, wantHashCalls: 1, wantTransformerClosed: true, @@ -152,7 +155,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, { name: "when multiple health checks are present previous config shouldn't be changed", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 1, wantHashCalls: 1, wantTransformerClosed: true, @@ -179,7 +182,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, { name: "when invalid health check URL is provided previous config shouldn't be changed", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 1, wantHashCalls: 1, wantTransformerClosed: true, @@ -202,7 +205,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, { name: "when config is not updated transformers are closed correctly", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 1, wantHashCalls: 1, wantTransformerClosed: true, @@ -220,13 +223,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, }, // hash of initial "testdata/ec_config.yaml" config file before reloading - EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + EncryptionFileContentHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", }, nil }, }, { name: "when config hash is not updated transformers are closed correctly", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 0, wantHashCalls: 1, wantTransformerClosed: true, @@ -234,7 +237,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash wantAddRateLimitedCount: 0, mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) { // hash of initial "testdata/ec_config.yaml" config file before reloading - return "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil + return "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil }, mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) { return nil, fmt.Errorf("should not be called") @@ -242,7 +245,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash }, { name: "when config hash errors transformers are closed correctly", - wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", + wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", wantLoadCalls: 0, wantHashCalls: 1, wantTransformerClosed: true, @@ -332,7 +335,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash } if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(test.wantMetrics), - "apiserver_encryption_config_controller_automatic_reloads_total", + "apiserver_encryption_config_controller_automatic_reloads_total", "apiserver_encryption_config_controller_automatic_reload_last_config_info", ); err != nil { t.Errorf("failed to validate metrics: %v", err) } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go index 43b6ebbf3f7..ed2d5c9a8fd 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go @@ -22,6 +22,7 @@ import ( "hash" "sync" + "k8s.io/apiserver/pkg/util/configmetrics" "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" ) @@ -53,10 +54,20 @@ var ( }, []string{"status", "apiserver_id_hash"}, ) + + encryptionConfigLastConfigInfo = metrics.NewDesc( + metrics.BuildFQName(namespace, subsystem, "last_config_info"), + "Information about the last applied encryption configuration with hash as label, split by apiserver identity.", + []string{"apiserver_id_hash", "hash"}, + nil, + metrics.ALPHA, + "", + ) ) var registerMetrics sync.Once var hashPool *sync.Pool +var configHashProvider = configmetrics.NewAtomicHashProvider() func RegisterMetrics() { registerMetrics.Do(func() { @@ -67,25 +78,37 @@ func RegisterMetrics() { } legacyregistry.MustRegister(encryptionConfigAutomaticReloadsTotal) legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds) + legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(encryptionConfigLastConfigInfo, configHashProvider)) }) } +func ResetMetricsForTest() { + encryptionConfigAutomaticReloadsTotal.Reset() + encryptionConfigAutomaticReloadLastTimestampSeconds.Reset() +} + func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) { apiServerIDHash := getHash(apiServerID) encryptionConfigAutomaticReloadsTotal.WithLabelValues("failure", apiServerIDHash).Inc() recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash) } -func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) { +func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID, encryptionConfigDataHash string) { apiServerIDHash := getHash(apiServerID) encryptionConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc() recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash) + + RecordEncryptionConfigLastConfigInfo(apiServerID, encryptionConfigDataHash) } func recordEncryptionConfigAutomaticReloadTimestamp(result, apiServerIDHash string) { encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result, apiServerIDHash).SetToCurrentTime() } +func RecordEncryptionConfigLastConfigInfo(apiServerID, encryptionConfigDataHash string) { + configHashProvider.SetHashes(getHash(apiServerID), encryptionConfigDataHash) +} + func getHash(data string) string { if len(data) == 0 { return "" diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go index 281ed96088b..6ef75196247 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics_test.go @@ -27,6 +27,7 @@ import ( const ( testAPIServerID = "testAPIServerID" testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37" + testConfigDataHash = "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3" ) func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) { @@ -53,16 +54,19 @@ func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) { # HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity. # TYPE apiserver_encryption_config_controller_automatic_reloads_total counter apiserver_encryption_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1 + # HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity. + # TYPE apiserver_encryption_config_controller_last_config_info gauge + apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3"} 1 ` metricNames := []string{ - namespace + "_" + subsystem + "_automatic_reload_success_total", namespace + "_" + subsystem + "_automatic_reloads_total", + namespace + "_" + subsystem + "_last_config_info", } - encryptionConfigAutomaticReloadsTotal.Reset() + ResetMetricsForTest() RegisterMetrics() - RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID) + RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, testConfigDataHash) if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metricNames...); err != nil { t.Fatal(err) } @@ -108,3 +112,41 @@ func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) { } } } + +func TestRecordEncryptionConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) { + ResetMetricsForTest() + RegisterMetrics() + + // Record first config + firstConfigHash := "sha256:firsthash" + RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, firstConfigHash) + + // Verify first config metric exists + firstExpected := ` + # HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity. + # TYPE apiserver_encryption_config_controller_last_config_info gauge + apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:firsthash"} 1 + ` + metricNames := []string{ + namespace + "_" + subsystem + "_last_config_info", + } + + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(firstExpected), metricNames...); err != nil { + t.Fatal(err) + } + + // Record second config - should clean up first config's metric + secondConfigHash := "sha256:secondhash" + RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, secondConfigHash) + + // Verify only second config metric exists + secondExpected := ` + # HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity. + # TYPE apiserver_encryption_config_controller_last_config_info gauge + apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:secondhash"} 1 + ` + + if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(secondExpected), metricNames...); err != nil { + t.Fatal(err) + } +} diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go index 8f73749c5f0..f5222569363 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go @@ -37,6 +37,7 @@ import ( "k8s.io/apiserver/pkg/server/healthz" "k8s.io/apiserver/pkg/server/options/encryptionconfig" encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller" + encryptionconfigmetrics "k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics" serverstorage "k8s.io/apiserver/pkg/server/storage" "k8s.io/apiserver/pkg/storage/etcd3/metrics" "k8s.io/apiserver/pkg/storage/storagebackend" @@ -298,6 +299,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro if err != nil { return err } + encryptionconfigmetrics.RecordEncryptionConfigLastConfigInfo(c.APIServerID, encryptionConfiguration.EncryptionFileContentHash) if s.EncryptionProviderConfigAutomaticReload { // with reload=true we will always have 1 health check diff --git a/staging/src/k8s.io/apiserver/pkg/util/configmetrics/info_collector.go b/staging/src/k8s.io/apiserver/pkg/util/configmetrics/info_collector.go new file mode 100644 index 00000000000..58e7a1b8a10 --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/configmetrics/info_collector.go @@ -0,0 +1,92 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configmetrics + +import ( + "slices" + "sync/atomic" + + "k8s.io/component-base/metrics" + "k8s.io/utils/ptr" +) + +// HashProvider is an interface for getting the current config hash values +type HashProvider interface { + GetCurrentHashes() []string + SetHashes(hashes ...string) +} + +// AtomicHashProvider implements HashProvider using a single atomic pointer to a slice +type AtomicHashProvider struct { + hashes *atomic.Pointer[[]string] +} + +// NewAtomicHashProvider creates a new atomic hash provider +func NewAtomicHashProvider() *AtomicHashProvider { + p := &AtomicHashProvider{ + hashes: &atomic.Pointer[[]string]{}, + } + // Initialize with empty slice + p.hashes.Store(ptr.To([]string{})) + return p +} + +func (h *AtomicHashProvider) GetCurrentHashes() []string { + hashesPtr := h.hashes.Load() + if hashesPtr == nil { + // should never happen, but just in case + return []string{} + } + // Return a copy to prevent external modification + hashes := *hashesPtr + return slices.Clone(hashes) +} + +func (h *AtomicHashProvider) SetHashes(hashes ...string) { + hashCopy := slices.Clone(hashes) + h.hashes.Store(&hashCopy) +} + +// NewConfigInfoCustomCollector creates a custom collector for config hash info metrics. +// This eliminates the need for state management and locks by collecting metrics on demand. +func NewConfigInfoCustomCollector(desc *metrics.Desc, hashProvider HashProvider) metrics.StableCollector { + return &configInfoCustomCollector{ + desc: desc, + hashProvider: hashProvider, + } +} + +type configInfoCustomCollector struct { + metrics.BaseStableCollector + desc *metrics.Desc + hashProvider HashProvider +} + +var _ metrics.StableCollector = &configInfoCustomCollector{} + +func (c *configInfoCustomCollector) DescribeWithStability(ch chan<- *metrics.Desc) { + ch <- c.desc +} + +func (c *configInfoCustomCollector) CollectWithStability(ch chan<- metrics.Metric) { + hashes := c.hashProvider.GetCurrentHashes() + if len(hashes) == 0 { + return + } + + ch <- metrics.NewLazyConstMetric(c.desc, metrics.GaugeValue, 1, hashes...) +} diff --git a/staging/src/k8s.io/apiserver/pkg/util/configmetrics/info_collector_test.go b/staging/src/k8s.io/apiserver/pkg/util/configmetrics/info_collector_test.go new file mode 100644 index 00000000000..87900e4290e --- /dev/null +++ b/staging/src/k8s.io/apiserver/pkg/util/configmetrics/info_collector_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configmetrics + +import ( + "testing" +) + +// assertHashesEqual is a test helper that compares expected and actual hash slices +func assertHashesEqual(t *testing.T, expected, actual []string) { + t.Helper() + if len(actual) != len(expected) { + t.Errorf("expected %d hashes, got %d", len(expected), len(actual)) + return + } + for i, hash := range actual { + if hash != expected[i] { + t.Errorf("expected hash %q at index %d, got %q", expected[i], i, hash) + } + } +} + +func TestAtomicHashProvider(t *testing.T) { + provider := NewAtomicHashProvider() + + apiServerIDHash := "sha256:abc123" + configHash := "sha256:hash1" + + // Test initial state + hashes := provider.GetCurrentHashes() + assertHashesEqual(t, []string{}, hashes) + + // Test setting hashes + provider.SetHashes(apiServerIDHash, configHash) + hashes = provider.GetCurrentHashes() + assertHashesEqual(t, []string{apiServerIDHash, configHash}, hashes) + + // Test updating hashes + newConfigHash := "sha256:hash2" + provider.SetHashes(apiServerIDHash, newConfigHash) + hashes = provider.GetCurrentHashes() + assertHashesEqual(t, []string{apiServerIDHash, newConfigHash}, hashes) + + // Test empty hashes + provider.SetHashes() + hashes = provider.GetCurrentHashes() + assertHashesEqual(t, []string{}, hashes) +} diff --git a/test/integration/apiserver/oidc/oidc_test.go b/test/integration/apiserver/oidc/oidc_test.go index 0da86b67020..c0b3d164908 100644 --- a/test/integration/apiserver/oidc/oidc_test.go +++ b/test/integration/apiserver/oidc/oidc_test.go @@ -22,6 +22,7 @@ import ( "crypto/elliptic" "crypto/rand" "crypto/rsa" + "crypto/sha256" "crypto/tls" "crypto/x509" "encoding/json" @@ -1146,6 +1147,7 @@ jwt: wantMetricStrings: []string{ `apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`, `apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`, + `apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`, }, }, { @@ -1229,6 +1231,7 @@ jwt: wantMetricStrings: []string{ `apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`, `apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`, + `apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`, }, }, { @@ -1277,6 +1280,7 @@ jwt: wantMetricStrings: []string{ `apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`, `apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`, + `apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`, }, }, { @@ -1332,6 +1336,7 @@ jwt: wantMetricStrings: []string{ `apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`, `apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`, + `apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`, }, }, { @@ -1390,6 +1395,7 @@ jwt: wantMetricStrings: []string{ `apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} FP`, `apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} 1`, + `apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_old_config_hash"} 1`, }, }, { @@ -1433,6 +1439,7 @@ kind: AuthenticationConfiguration wantMetricStrings: []string{ `apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`, `apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`, + `apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`, }, }, { @@ -1490,6 +1497,7 @@ jwt: wantMetricStrings: []string{ `apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} FP`, `apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} 1`, + `apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_old_config_hash"} 1`, }, }, } @@ -1529,8 +1537,9 @@ jwt: _ = tempFile.Close() }() + newAuthConfig := tt.newAuthConfigFn(t, oidcServer.URL(), string(caCert)) // Write the new content to the temporary file - _, err = tempFile.Write([]byte(tt.newAuthConfigFn(t, oidcServer.URL(), string(caCert)))) + _, err = tempFile.Write([]byte(newAuthConfig)) require.NoError(t, err) // Atomically replace the original file with the temporary file @@ -1565,6 +1574,16 @@ jwt: _, err = client.CoreV1().Pods(defaultNamespace).List(ctx, metav1.ListOptions{}) tt.newAssertErrFn(t, err) + oldAuthConfigHash := getHash(tt.authConfigFn(t, oidcServer.URL(), string(caCert))) + newAuthConfigHash := getHash(newAuthConfig) + for i := range tt.wantMetricStrings { + if strings.Contains(tt.wantMetricStrings[i], "replace_with_new_config_hash") { + tt.wantMetricStrings[i] = strings.ReplaceAll(tt.wantMetricStrings[i], "replace_with_new_config_hash", newAuthConfigHash) + } else if strings.Contains(tt.wantMetricStrings[i], "replace_with_old_config_hash") { + tt.wantMetricStrings[i] = strings.ReplaceAll(tt.wantMetricStrings[i], "replace_with_old_config_hash", oldAuthConfigHash) + } + } + adminClient := kubernetes.NewForConfigOrDie(apiServer.ClientConfig) body, err := adminClient.RESTClient().Get().AbsPath("/metrics").DoRaw(ctx) require.NoError(t, err) @@ -2114,3 +2133,10 @@ func testContext(t *testing.T) context.Context { t.Cleanup(cancel) return ctx } + +func getHash(data string) string { + if len(data) == 0 { + return "" + } + return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data))) +} diff --git a/test/integration/auth/authz_config_test.go b/test/integration/auth/authz_config_test.go index 0dd2d75183b..96fd331364c 100644 --- a/test/integration/auth/authz_config_test.go +++ b/test/integration/auth/authz_config_test.go @@ -735,8 +735,8 @@ authorizers: if err != nil { t.Fatal(err) } - if initialMetrics.reloadSuccess == nil { - t.Fatal("expected success timestamp, got none") + if initialMetrics.reloadSuccess != nil { + t.Fatal("expected no success timestamp, got one") } if initialMetrics.reloadFailure != nil { t.Fatal("expected no failure timestamp, got one") @@ -754,19 +754,13 @@ authorizers: if err != nil { t.Fatal(err) } - if reload1Metrics.reloadSuccess == nil { - t.Fatal("expected success timestamp, got none") - } - if !reload1Metrics.reloadSuccess.Equal(*initialMetrics.reloadSuccess) { - t.Fatalf("success timestamp changed from initial success %s to %s unexpectedly", initialMetrics.reloadSuccess.String(), reload1Metrics.reloadSuccess.String()) + if reload1Metrics.reloadSuccess != nil { + t.Fatal("expected no success timestamp, got one") } if reload1Metrics.reloadFailure == nil { t.Log("expected failure timestamp, got nil, retrying") return false, nil } - if !reload1Metrics.reloadFailure.After(*reload1Metrics.reloadSuccess) { - t.Fatalf("expected failure timestamp to be more recent than success timestamp, got %s <= %s", reload1Metrics.reloadFailure.String(), reload1Metrics.reloadSuccess.String()) - } return true, nil }) if err != nil { @@ -829,15 +823,19 @@ authorizers: t.Fatalf("failure timestamp changed from reload1Metrics.reloadFailure %s to %s unexpectedly", reload1Metrics.reloadFailure.String(), reload2Metrics.reloadFailure.String()) } if reload2Metrics.reloadSuccess == nil { - t.Fatal("expected success timestamp, got none") - } - if reload2Metrics.reloadSuccess.Equal(*initialMetrics.reloadSuccess) { - t.Log("success timestamp hasn't updated from initial success, retrying") + t.Log("expected success timestamp, got nil, retrying") return false, nil } if !reload2Metrics.reloadSuccess.After(*reload2Metrics.reloadFailure) { t.Fatalf("expected success timestamp to be more recent than failure, got %s <= %s", reload2Metrics.reloadSuccess.String(), reload2Metrics.reloadFailure.String()) } + if len(reload2Metrics.configHash) == 0 { + t.Fatal("expected config hash, got none") + } + if reload2Metrics.configHash == initialMetrics.configHash { + t.Logf("config hash %s is the same as initial %s, retrying", reload2Metrics.configHash, initialMetrics.configHash) + return false, nil + } return true, nil }) if err != nil { @@ -883,6 +881,12 @@ authorizers: if !reload3Metrics.reloadSuccess.Equal(*reload2Metrics.reloadSuccess) { t.Fatalf("success timestamp changed from %s to %s unexpectedly", reload2Metrics.reloadSuccess.String(), reload3Metrics.reloadSuccess.String()) } + if len(reload3Metrics.configHash) == 0 { + t.Fatal("expected config hash, got none") + } + if reload3Metrics.configHash != reload2Metrics.configHash { + t.Fatalf("expected config hash to be the same as reload2Metrics %s, got %s", reload2Metrics.configHash, reload3Metrics.configHash) + } if reload3Metrics.reloadFailure == nil { t.Log("expected failure timestamp, got nil, retrying") return false, nil @@ -925,6 +929,7 @@ authorizers: type metrics struct { reloadSuccess *time.Time reloadFailure *time.Time + configHash string decisions map[authorizerKey]map[string]int exclusions int evalErrors int @@ -945,12 +950,14 @@ var webhookMatchConditionEvalErrorMetric = regexp.MustCompile(`apiserver_authori var whTotalMetric = regexp.MustCompile(`apiserver_authorization_webhook_evaluations_total{name="(.*?)",result="(.*?)"} (\d+)`) var webhookDurationMetric = regexp.MustCompile(`apiserver_authorization_webhook_duration_seconds_count{name="(.*?)",result="(.*?)"} (\d+)`) var webhookFailOpenMetric = regexp.MustCompile(`apiserver_authorization_webhook_evaluations_fail_open_total{name="(.*?)",result="(.*?)"} (\d+)`) +var configInfoMetric = regexp.MustCompile(`apiserver_authorization_config_controller_last_config_info\{apiserver_id_hash="sha256:[^"]*",hash="([^"]*)"\} (\d+)`) func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) { data, err := client.RESTClient().Get().AbsPath("/metrics").DoRaw(context.TODO()) // apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",status="success"} 1.7002567356895502e+09 // apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",status="failure"} 1.7002567356895502e+09 + // apiserver_authorization_config_controller_automatic_reload_last_config_info{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1 // apiserver_authorization_decisions_total{decision="allowed",name="allow.example.com",type="Webhook"} 2 // apiserver_authorization_decisions_total{decision="allowed",name="allowreloaded.example.com",type="Webhook"} 1 // apiserver_authorization_decisions_total{decision="denied",name="deny.example.com",type="Webhook"} 1 @@ -1052,6 +1059,12 @@ func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) { t.Log("failure", m.reloadFailure.String()) } } + if matches := configInfoMetric.FindStringSubmatch(line); matches != nil { + t.Logf("line: %s\n", line) + t.Log(matches) + m.configHash = matches[1] + t.Logf("config hash: %s", m.configHash) + } } return &m, nil } diff --git a/test/integration/controlplane/transformation/kms_transformation_test.go b/test/integration/controlplane/transformation/kms_transformation_test.go index eeb0de83850..89c539dd24b 100644 --- a/test/integration/controlplane/transformation/kms_transformation_test.go +++ b/test/integration/controlplane/transformation/kms_transformation_test.go @@ -23,6 +23,7 @@ import ( "bytes" "context" "crypto/aes" + "crypto/sha256" "encoding/base64" "encoding/binary" "fmt" @@ -352,13 +353,6 @@ resources: t.Fatal(err) } - // assert that the metrics we collect during the test run match expectations - // NOTE: 2 successful automatic reload resulted from 2 config file updates - wantMetricStrings := []string{ - `apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`, - `apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 2`, - } - test.secret, err = test.createSecret(testSecret, testNamespace) if err != nil { t.Fatalf("Failed to create test secret, error: %v", err) @@ -582,6 +576,15 @@ resources: if err != nil { t.Fatal(err) } + + // assert that the metrics we collect during the test run match expectations + // NOTE: 2 successful automatic reload resulted from 2 config file updates + wantMetricStrings := []string{ + `apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`, + `apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 2`, + fmt.Sprintf(`apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="%s"} 1`, getHash(encryptionConfigWithoutOldProvider)), + } + defer func() { body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx) if err != nil { @@ -1205,3 +1208,10 @@ resources: // the healthz check should be OK. mustBeHealthy(t, "/kms-providers", "ok", test.kubeAPIServer.ClientConfig) } + +func getHash(data string) string { + if len(data) == 0 { + return "" + } + return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data))) +}