Merge pull request #132299 from aramase/aramase/f/authn_config_hash_metric

Add `last_config_info` metric for authn, authz and encryption config
This commit is contained in:
Kubernetes Prow Robot
2025-07-17 18:24:24 -07:00
committed by GitHub
20 changed files with 483 additions and 62 deletions

View File

@@ -68,6 +68,9 @@ type Config struct {
// AuthorizationConfiguration stores the configuration for the Authorizer chain
// It will deprecate most of the above flags when GA
AuthorizationConfiguration *authzconfig.AuthorizationConfiguration
// InitialAuthorizationConfigurationData holds the initial authorization configuration data
// that was read from the authorization configuration file.
InitialAuthorizationConfigurationData string
}
// New returns the right sort of union of multiple authorizer.Authorizer objects
@@ -84,6 +87,7 @@ func (config Config) New(ctx context.Context, serverID string) (authorizer.Autho
initialConfig: config,
apiServerID: serverID,
lastLoadedConfig: config.AuthorizationConfiguration,
lastReadData: []byte(config.InitialAuthorizationConfigurationData),
reloadInterval: time.Minute,
compiler: authorizationcel.NewDefaultCompiler(),
}
@@ -160,12 +164,16 @@ func GetNameForAuthorizerMode(mode string) string {
return strings.ToLower(mode)
}
func LoadAndValidateFile(configFile string, compiler authorizationcel.Compiler, requireNonWebhookTypes sets.Set[authzconfig.AuthorizerType]) (*authzconfig.AuthorizationConfiguration, error) {
func LoadAndValidateFile(configFile string, compiler authorizationcel.Compiler, requireNonWebhookTypes sets.Set[authzconfig.AuthorizerType]) (*authzconfig.AuthorizationConfiguration, string, error) {
data, err := os.ReadFile(configFile)
if err != nil {
return nil, err
return nil, "", err
}
return LoadAndValidateData(data, compiler, requireNonWebhookTypes)
config, err := LoadAndValidateData(data, compiler, requireNonWebhookTypes)
if err != nil {
return nil, "", err
}
return config, string(data), nil
}
func LoadAndValidateData(data []byte, compiler authorizationcel.Compiler, requireNonWebhookTypes sets.Set[authzconfig.AuthorizerType]) (*authzconfig.AuthorizationConfiguration, error) {
@@ -198,7 +206,6 @@ func LoadAndValidateData(data []byte, compiler authorizationcel.Compiler, requir
if expectedName != authorizer.Name {
allErrors = append(allErrors, fmt.Errorf("expected name %s for authorizer %s instead of %s", expectedName, authorizer.Type, authorizer.Name))
}
}
if missingTypes := requireNonWebhookTypes.Difference(seenModes); missingTypes.Len() > 0 {

View File

@@ -192,7 +192,7 @@ type kubeapiserverWebhookMetrics struct {
// Blocks until ctx is complete.
func (r *reloadableAuthorizerResolver) runReload(ctx context.Context) {
metrics.RegisterMetrics()
metrics.RecordAuthorizationConfigAutomaticReloadSuccess(r.apiServerID)
metrics.RecordAuthorizationConfigLastConfigInfo(r.apiServerID, string(r.lastReadData))
filesystem.WatchUntil(
ctx,
@@ -250,5 +250,5 @@ func (r *reloadableAuthorizerResolver) checkFile(ctx context.Context) {
ruleResolver: ruleResolver,
})
klog.InfoS("reloaded authz config")
metrics.RecordAuthorizationConfigAutomaticReloadSuccess(r.apiServerID)
metrics.RecordAuthorizationConfigAutomaticReloadSuccess(r.apiServerID, string(data))
}

View File

@@ -727,6 +727,7 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(
if len(o.AuthenticationConfigFile) > 0 {
authenticationconfigmetrics.RegisterMetrics()
authenticationconfigmetrics.RecordAuthenticationConfigLastConfigInfo(apiServerID, authenticatorConfig.AuthenticationConfigData)
trackedAuthenticationConfigData := authenticatorConfig.AuthenticationConfigData
var mu sync.Mutex
@@ -789,7 +790,7 @@ func (o *BuiltInAuthenticationOptions) ApplyTo(
trackedAuthenticationConfigData = authConfigData
klog.InfoS("reloaded authentication config")
authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID)
authenticationconfigmetrics.RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID, authConfigData)
},
func(err error) { klog.ErrorS(err, "watching authentication config file") },
)

View File

@@ -118,7 +118,7 @@ func (o *BuiltInAuthorizationOptions) Validate() []error {
}
// load/validate kube-apiserver authz config with no opinion about required modes
_, err := authorizer.LoadAndValidateFile(o.AuthorizationConfigurationFile, authorizationcel.NewDefaultCompiler(), nil)
_, _, err := authorizer.LoadAndValidateFile(o.AuthorizationConfigurationFile, authorizationcel.NewDefaultCompiler(), nil)
if err != nil {
return append(allErrors, err)
}
@@ -219,6 +219,7 @@ func (o *BuiltInAuthorizationOptions) ToAuthorizationConfig(versionedInformerFac
var authorizationConfiguration *authzconfig.AuthorizationConfiguration
var err error
var authorizationConfigData string
// if --authorization-config is set, check if
// - the feature flag is set
@@ -236,7 +237,7 @@ func (o *BuiltInAuthorizationOptions) ToAuthorizationConfig(versionedInformerFac
return nil, fmt.Errorf("--%s can not be specified when --%s or --authorization-webhook-* flags are defined", authorizationConfigFlag, authorizationModeFlag)
}
// load/validate kube-apiserver authz config with no opinion about required modes
authorizationConfiguration, err = authorizer.LoadAndValidateFile(o.AuthorizationConfigurationFile, authorizationcel.NewDefaultCompiler(), nil)
authorizationConfiguration, authorizationConfigData, err = authorizer.LoadAndValidateFile(o.AuthorizationConfigurationFile, authorizationcel.NewDefaultCompiler(), nil)
if err != nil {
return nil, err
}
@@ -252,8 +253,9 @@ func (o *BuiltInAuthorizationOptions) ToAuthorizationConfig(versionedInformerFac
VersionedInformerFactory: versionedInformerFactory,
WebhookRetryBackoff: o.WebhookRetryBackoff,
ReloadFile: o.AuthorizationConfigurationFile,
AuthorizationConfiguration: authorizationConfiguration,
ReloadFile: o.AuthorizationConfigurationFile,
AuthorizationConfiguration: authorizationConfiguration,
InitialAuthorizationConfigurationData: authorizationConfigData,
}, nil
}

View File

@@ -21,6 +21,7 @@ import (
"fmt"
"sync"
"k8s.io/apiserver/pkg/util/configmetrics"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
@@ -52,14 +53,25 @@ var (
},
[]string{"status", "apiserver_id_hash"},
)
authenticationConfigLastConfigInfo = metrics.NewDesc(
metrics.BuildFQName(namespace, subsystem, "last_config_info"),
"Information about the last applied authentication configuration with hash as label, split by apiserver identity.",
[]string{"apiserver_id_hash", "hash"},
nil,
metrics.ALPHA,
"",
)
)
var registerMetrics sync.Once
var configHashProvider = configmetrics.NewAtomicHashProvider()
func RegisterMetrics() {
registerMetrics.Do(func() {
legacyregistry.MustRegister(authenticationConfigAutomaticReloadsTotal)
legacyregistry.MustRegister(authenticationConfigAutomaticReloadLastTimestampSeconds)
legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(authenticationConfigLastConfigInfo, configHashProvider))
})
}
@@ -74,10 +86,16 @@ func RecordAuthenticationConfigAutomaticReloadFailure(apiServerID string) {
authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime()
}
func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID string) {
func RecordAuthenticationConfigAutomaticReloadSuccess(apiServerID, authConfigData string) {
apiServerIDHash := getHash(apiServerID)
authenticationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
authenticationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime()
RecordAuthenticationConfigLastConfigInfo(apiServerID, authConfigData)
}
func RecordAuthenticationConfigLastConfigInfo(apiServerID, authConfigData string) {
configHashProvider.SetHashes(getHash(apiServerID), getHash(authConfigData))
}
func getHash(data string) string {

View File

@@ -27,6 +27,18 @@ import (
const (
testAPIServerID = "testAPIServerID"
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
testConfigData = `
apiVersion: apiserver.config.k8s.io/v1
kind: AuthenticationConfiguration
jwt:
- issuer:
url: https://test-issuer
audiences: [ "aud" ]
claimMappings:
username:
claim: sub
prefix: ""
`
)
func TestRecordAuthenticationConfigAutomaticReloadFailure(t *testing.T) {
@@ -53,15 +65,19 @@ func TestRecordAuthenticationConfigAutomaticReloadSuccess(t *testing.T) {
# HELP apiserver_authentication_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authentication configuration split by status and apiserver identity.
# TYPE apiserver_authentication_config_controller_automatic_reloads_total counter
apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
# HELP apiserver_authentication_config_controller_last_config_info [ALPHA] Information about the last applied authentication configuration with hash as label, split by apiserver identity.
# TYPE apiserver_authentication_config_controller_last_config_info gauge
apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:ccbcaf98557c273dfc779222d54a5bd3e785ea5330048f3bf4278cf3997b669c"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
authenticationConfigAutomaticReloadsTotal.Reset()
ResetMetricsForTest()
RegisterMetrics()
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID)
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, testConfigData)
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
@@ -107,3 +123,34 @@ func TestAuthenticationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
}
}
}
func TestRecordAuthenticationConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) {
ResetMetricsForTest()
RegisterMetrics()
// Record initial success with first config
firstConfig := "config1"
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, firstConfig)
// Record success with different config - should clean up old metric
secondConfig := "config2"
RecordAuthenticationConfigAutomaticReloadSuccess(testAPIServerID, secondConfig)
// Verify only the latest hash is present
expectedValue := `
# HELP apiserver_authentication_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authentication configuration split by status and apiserver identity.
# TYPE apiserver_authentication_config_controller_automatic_reloads_total counter
apiserver_authentication_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 2
# HELP apiserver_authentication_config_controller_last_config_info [ALPHA] Information about the last applied authentication configuration with hash as label, split by apiserver identity.
# TYPE apiserver_authentication_config_controller_last_config_info gauge
apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
}

View File

@@ -22,6 +22,7 @@ import (
"hash"
"sync"
"k8s.io/apiserver/pkg/util/configmetrics"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
@@ -53,10 +54,20 @@ var (
},
[]string{"status", "apiserver_id_hash"},
)
authorizationConfigLastConfigInfo = metrics.NewDesc(
metrics.BuildFQName(namespace, subsystem, "last_config_info"),
"Information about the last applied authorization configuration with hash as label, split by apiserver identity.",
[]string{"apiserver_id_hash", "hash"},
nil,
metrics.ALPHA,
"",
)
)
var registerMetrics sync.Once
var hashPool *sync.Pool
var configHashProvider = configmetrics.NewAtomicHashProvider()
func RegisterMetrics() {
registerMetrics.Do(func() {
@@ -67,6 +78,7 @@ func RegisterMetrics() {
}
legacyregistry.MustRegister(authorizationConfigAutomaticReloadsTotal)
legacyregistry.MustRegister(authorizationConfigAutomaticReloadLastTimestampSeconds)
legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(authorizationConfigLastConfigInfo, configHashProvider))
})
}
@@ -81,10 +93,16 @@ func RecordAuthorizationConfigAutomaticReloadFailure(apiServerID string) {
authorizationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("failure", apiServerIDHash).SetToCurrentTime()
}
func RecordAuthorizationConfigAutomaticReloadSuccess(apiServerID string) {
func RecordAuthorizationConfigAutomaticReloadSuccess(apiServerID, authzConfigData string) {
apiServerIDHash := getHash(apiServerID)
authorizationConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
authorizationConfigAutomaticReloadLastTimestampSeconds.WithLabelValues("success", apiServerIDHash).SetToCurrentTime()
RecordAuthorizationConfigLastConfigInfo(apiServerID, authzConfigData)
}
func RecordAuthorizationConfigLastConfigInfo(apiServerID, authzConfigData string) {
configHashProvider.SetHashes(getHash(apiServerID), getHash(authzConfigData))
}
func getHash(data string) string {

View File

@@ -27,6 +27,27 @@ import (
const (
testAPIServerID = "testAPIServerID"
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
testConfigData = `
apiVersion: apiserver.config.k8s.io/v1
kind: AuthorizationConfiguration
authorizers:
- type: Webhook
name: error.example.com
webhook:
timeout: 5s
failurePolicy: Deny
subjectAccessReviewVersion: v1
matchConditionSubjectAccessReviewVersion: v1
authorizedTTL: 1ms
unauthorizedTTL: 1ms
connectionInfo:
type: KubeConfigFile
kubeConfigFile: /path/to/kubeconfig
matchConditions:
- expression: has(request.resourceAttributes)
- expression: 'request.resourceAttributes.namespace == "fail"'
- expression: 'request.resourceAttributes.name == "error"'
`
)
func TestRecordAuthorizationConfigAutomaticReloadFailure(t *testing.T) {
@@ -53,15 +74,19 @@ func TestRecordAuthorizationConfigAutomaticReloadSuccess(t *testing.T) {
# HELP apiserver_authorization_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authorization configuration split by status and apiserver identity.
# TYPE apiserver_authorization_config_controller_automatic_reloads_total counter
apiserver_authorization_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
# HELP apiserver_authorization_config_controller_last_config_info [ALPHA] Information about the last applied authorization configuration with hash as label, split by apiserver identity.
# TYPE apiserver_authorization_config_controller_last_config_info gauge
apiserver_authorization_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:2a66c95e5593fc74e6c3dbbb708741361f0690ed45d17e4d4ac0b9c282b6538f"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
authorizationConfigAutomaticReloadsTotal.Reset()
ResetMetricsForTest()
RegisterMetrics()
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID)
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, testConfigData)
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
@@ -107,3 +132,34 @@ func TestAuthorizationConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
}
}
}
func TestRecordAuthorizationConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) {
ResetMetricsForTest()
RegisterMetrics()
// Record initial success with first config
firstConfig := "config1"
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, firstConfig)
// Record success with different config - should clean up old metric
secondConfig := "config2"
RecordAuthorizationConfigAutomaticReloadSuccess(testAPIServerID, secondConfig)
// Verify only the latest hash is present
expectedValue := `
# HELP apiserver_authorization_config_controller_automatic_reloads_total [BETA] Total number of automatic reloads of authorization configuration split by status and apiserver identity.
# TYPE apiserver_authorization_config_controller_automatic_reloads_total counter
apiserver_authorization_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 2
# HELP apiserver_authorization_config_controller_last_config_info [ALPHA] Information about the last applied authorization configuration with hash as label, split by apiserver identity.
# TYPE apiserver_authorization_config_controller_last_config_info gauge
apiserver_authorization_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1
`
metrics := []string{
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metrics...); err != nil {
t.Fatal(err)
}
}

View File

@@ -907,9 +907,8 @@ func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte,
// computeEncryptionConfigHash returns the expected hash for an encryption config file that has been loaded as bytes.
// We use a hash instead of the raw file contents when tracking changes to avoid holding any encryption keys in memory outside of their associated transformers.
// This hash must be used in-memory and not externalized to the process because it has no cross-release stability guarantees.
func computeEncryptionConfigHash(data []byte) string {
return fmt.Sprintf("k8s:enc:unstable:1:%x", sha256.Sum256(data))
return fmt.Sprintf("sha256:%x", sha256.Sum256(data))
}
var _ storagevalue.ResourceTransformers = &DynamicTransformers{}

View File

@@ -1843,7 +1843,7 @@ func errString(err error) string {
func TestComputeEncryptionConfigHash(t *testing.T) {
// hash the empty string to be sure that sha256 is being used
expect := "k8s:enc:unstable:1:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
expect := "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
sum := computeEncryptionConfigHash([]byte(""))
if expect != sum {
t.Errorf("expected hash %q but got %q", expect, sum)
@@ -2220,7 +2220,7 @@ func TestGetEncryptionConfigHash(t *testing.T) {
{
name: "valid file",
filepath: "testdata/valid-configs/secret-box-first.yaml",
wantHash: "k8s:enc:unstable:1:c638c0327dbc3276dd1fcf3e67895d19ebca16b91ae0d19af24ef0759b8e0f66",
wantHash: "sha256:c638c0327dbc3276dd1fcf3e67895d19ebca16b91ae0d19af24ef0759b8e0f66",
wantErr: ``,
},
}

View File

@@ -174,7 +174,7 @@ func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Conte
}
if updatedEffectiveConfig && err == nil {
metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID)
metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID, encryptionConfiguration.EncryptionFileContentHash)
}
if err != nil {

View File

@@ -47,6 +47,9 @@ func TestController(t *testing.T) {
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
# TYPE apiserver_encryption_config_controller_automatic_reloads_total counter
apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",status="success"} 1
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
# TYPE apiserver_encryption_config_controller_last_config_info gauge
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:cd8a60cec6134082e9f37e7a4146b4bc14a0bf8a863237c36ec8fdb658c3e027",hash="sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef"} 1
`
const expectedFailureMetricValue = `
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
@@ -67,7 +70,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
}{
{
name: "when invalid config is provided previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@@ -82,7 +85,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when new valid config is provided it should be updated",
wantECFileHash: "some new config hash",
wantECFileHash: "sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef", // some new config hash
wantLoadCalls: 1,
wantHashCalls: 1,
wantMetrics: expectedSuccessMetricValue,
@@ -98,13 +101,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
err: nil,
},
},
EncryptionFileContentHash: "some new config hash",
EncryptionFileContentHash: "sha256:6f6af143a5aec5c4056d759f2bdf8b6ffe218a2bf3846c4fd42b6baef037c5ef", // some new config hash
}, nil
},
},
{
name: "when same valid config is provided previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@@ -122,13 +125,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
},
// hash of initial "testdata/ec_config.yaml" config file before reloading
EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
EncryptionFileContentHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
}, nil
},
},
{
name: "when transformer's health check fails previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@@ -152,7 +155,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when multiple health checks are present previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@@ -179,7 +182,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when invalid health check URL is provided previous config shouldn't be changed",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@@ -202,7 +205,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when config is not updated transformers are closed correctly",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 1,
wantHashCalls: 1,
wantTransformerClosed: true,
@@ -220,13 +223,13 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
},
// hash of initial "testdata/ec_config.yaml" config file before reloading
EncryptionFileContentHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
EncryptionFileContentHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
}, nil
},
},
{
name: "when config hash is not updated transformers are closed correctly",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 0,
wantHashCalls: 1,
wantTransformerClosed: true,
@@ -234,7 +237,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
wantAddRateLimitedCount: 0,
mockGetEncryptionConfigHash: func(ctx context.Context, filepath string) (string, error) {
// hash of initial "testdata/ec_config.yaml" config file before reloading
return "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil
return "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3", nil
},
mockLoadEncryptionConfig: func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error) {
return nil, fmt.Errorf("should not be called")
@@ -242,7 +245,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
},
{
name: "when config hash errors transformers are closed correctly",
wantECFileHash: "k8s:enc:unstable:1:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantECFileHash: "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3",
wantLoadCalls: 0,
wantHashCalls: 1,
wantTransformerClosed: true,
@@ -332,7 +335,7 @@ apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash
}
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(test.wantMetrics),
"apiserver_encryption_config_controller_automatic_reloads_total",
"apiserver_encryption_config_controller_automatic_reloads_total", "apiserver_encryption_config_controller_automatic_reload_last_config_info",
); err != nil {
t.Errorf("failed to validate metrics: %v", err)
}

View File

@@ -22,6 +22,7 @@ import (
"hash"
"sync"
"k8s.io/apiserver/pkg/util/configmetrics"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
)
@@ -53,10 +54,20 @@ var (
},
[]string{"status", "apiserver_id_hash"},
)
encryptionConfigLastConfigInfo = metrics.NewDesc(
metrics.BuildFQName(namespace, subsystem, "last_config_info"),
"Information about the last applied encryption configuration with hash as label, split by apiserver identity.",
[]string{"apiserver_id_hash", "hash"},
nil,
metrics.ALPHA,
"",
)
)
var registerMetrics sync.Once
var hashPool *sync.Pool
var configHashProvider = configmetrics.NewAtomicHashProvider()
func RegisterMetrics() {
registerMetrics.Do(func() {
@@ -67,25 +78,37 @@ func RegisterMetrics() {
}
legacyregistry.MustRegister(encryptionConfigAutomaticReloadsTotal)
legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
legacyregistry.CustomMustRegister(configmetrics.NewConfigInfoCustomCollector(encryptionConfigLastConfigInfo, configHashProvider))
})
}
func ResetMetricsForTest() {
encryptionConfigAutomaticReloadsTotal.Reset()
encryptionConfigAutomaticReloadLastTimestampSeconds.Reset()
}
func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) {
apiServerIDHash := getHash(apiServerID)
encryptionConfigAutomaticReloadsTotal.WithLabelValues("failure", apiServerIDHash).Inc()
recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash)
}
func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) {
func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID, encryptionConfigDataHash string) {
apiServerIDHash := getHash(apiServerID)
encryptionConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash)
RecordEncryptionConfigLastConfigInfo(apiServerID, encryptionConfigDataHash)
}
func recordEncryptionConfigAutomaticReloadTimestamp(result, apiServerIDHash string) {
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result, apiServerIDHash).SetToCurrentTime()
}
func RecordEncryptionConfigLastConfigInfo(apiServerID, encryptionConfigDataHash string) {
configHashProvider.SetHashes(getHash(apiServerID), encryptionConfigDataHash)
}
func getHash(data string) string {
if len(data) == 0 {
return ""

View File

@@ -27,6 +27,7 @@ import (
const (
testAPIServerID = "testAPIServerID"
testAPIServerIDHash = "sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37"
testConfigDataHash = "sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3"
)
func TestRecordEncryptionConfigAutomaticReloadFailure(t *testing.T) {
@@ -53,16 +54,19 @@ func TestRecordEncryptionConfigAutomaticReloadSuccess(t *testing.T) {
# HELP apiserver_encryption_config_controller_automatic_reloads_total [ALPHA] Total number of reload successes and failures of encryption configuration split by apiserver identity.
# TYPE apiserver_encryption_config_controller_automatic_reloads_total counter
apiserver_encryption_config_controller_automatic_reloads_total {apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",status="success"} 1
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
# TYPE apiserver_encryption_config_controller_last_config_info gauge
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:6bc9f4aa2e5587afbb96074e1809550cbc4de3cc3a35717dac8ff2800a147fd3"} 1
`
metricNames := []string{
namespace + "_" + subsystem + "_automatic_reload_success_total",
namespace + "_" + subsystem + "_automatic_reloads_total",
namespace + "_" + subsystem + "_last_config_info",
}
encryptionConfigAutomaticReloadsTotal.Reset()
ResetMetricsForTest()
RegisterMetrics()
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID)
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, testConfigDataHash)
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expectedValue), metricNames...); err != nil {
t.Fatal(err)
}
@@ -108,3 +112,41 @@ func TestEncryptionConfigAutomaticReloadLastTimestampSeconds(t *testing.T) {
}
}
}
func TestRecordEncryptionConfigAutomaticReloadSuccess_StaleMetricCleanup(t *testing.T) {
ResetMetricsForTest()
RegisterMetrics()
// Record first config
firstConfigHash := "sha256:firsthash"
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, firstConfigHash)
// Verify first config metric exists
firstExpected := `
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
# TYPE apiserver_encryption_config_controller_last_config_info gauge
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:firsthash"} 1
`
metricNames := []string{
namespace + "_" + subsystem + "_last_config_info",
}
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(firstExpected), metricNames...); err != nil {
t.Fatal(err)
}
// Record second config - should clean up first config's metric
secondConfigHash := "sha256:secondhash"
RecordEncryptionConfigAutomaticReloadSuccess(testAPIServerID, secondConfigHash)
// Verify only second config metric exists
secondExpected := `
# HELP apiserver_encryption_config_controller_last_config_info [ALPHA] Information about the last applied encryption configuration with hash as label, split by apiserver identity.
# TYPE apiserver_encryption_config_controller_last_config_info gauge
apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:14f9d63e669337ac6bfda2e2162915ee6a6067743eddd4e5c374b572f951ff37",hash="sha256:secondhash"} 1
`
if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(secondExpected), metricNames...); err != nil {
t.Fatal(err)
}
}

View File

@@ -37,6 +37,7 @@ import (
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
encryptionconfigmetrics "k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storage/etcd3/metrics"
"k8s.io/apiserver/pkg/storage/storagebackend"
@@ -298,6 +299,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
if err != nil {
return err
}
encryptionconfigmetrics.RecordEncryptionConfigLastConfigInfo(c.APIServerID, encryptionConfiguration.EncryptionFileContentHash)
if s.EncryptionProviderConfigAutomaticReload {
// with reload=true we will always have 1 health check

View File

@@ -0,0 +1,92 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package configmetrics
import (
"slices"
"sync/atomic"
"k8s.io/component-base/metrics"
"k8s.io/utils/ptr"
)
// HashProvider is an interface for getting the current config hash values
type HashProvider interface {
GetCurrentHashes() []string
SetHashes(hashes ...string)
}
// AtomicHashProvider implements HashProvider using a single atomic pointer to a slice
type AtomicHashProvider struct {
hashes *atomic.Pointer[[]string]
}
// NewAtomicHashProvider creates a new atomic hash provider
func NewAtomicHashProvider() *AtomicHashProvider {
p := &AtomicHashProvider{
hashes: &atomic.Pointer[[]string]{},
}
// Initialize with empty slice
p.hashes.Store(ptr.To([]string{}))
return p
}
func (h *AtomicHashProvider) GetCurrentHashes() []string {
hashesPtr := h.hashes.Load()
if hashesPtr == nil {
// should never happen, but just in case
return []string{}
}
// Return a copy to prevent external modification
hashes := *hashesPtr
return slices.Clone(hashes)
}
func (h *AtomicHashProvider) SetHashes(hashes ...string) {
hashCopy := slices.Clone(hashes)
h.hashes.Store(&hashCopy)
}
// NewConfigInfoCustomCollector creates a custom collector for config hash info metrics.
// This eliminates the need for state management and locks by collecting metrics on demand.
func NewConfigInfoCustomCollector(desc *metrics.Desc, hashProvider HashProvider) metrics.StableCollector {
return &configInfoCustomCollector{
desc: desc,
hashProvider: hashProvider,
}
}
type configInfoCustomCollector struct {
metrics.BaseStableCollector
desc *metrics.Desc
hashProvider HashProvider
}
var _ metrics.StableCollector = &configInfoCustomCollector{}
func (c *configInfoCustomCollector) DescribeWithStability(ch chan<- *metrics.Desc) {
ch <- c.desc
}
func (c *configInfoCustomCollector) CollectWithStability(ch chan<- metrics.Metric) {
hashes := c.hashProvider.GetCurrentHashes()
if len(hashes) == 0 {
return
}
ch <- metrics.NewLazyConstMetric(c.desc, metrics.GaugeValue, 1, hashes...)
}

View File

@@ -0,0 +1,62 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package configmetrics
import (
"testing"
)
// assertHashesEqual is a test helper that compares expected and actual hash slices
func assertHashesEqual(t *testing.T, expected, actual []string) {
t.Helper()
if len(actual) != len(expected) {
t.Errorf("expected %d hashes, got %d", len(expected), len(actual))
return
}
for i, hash := range actual {
if hash != expected[i] {
t.Errorf("expected hash %q at index %d, got %q", expected[i], i, hash)
}
}
}
func TestAtomicHashProvider(t *testing.T) {
provider := NewAtomicHashProvider()
apiServerIDHash := "sha256:abc123"
configHash := "sha256:hash1"
// Test initial state
hashes := provider.GetCurrentHashes()
assertHashesEqual(t, []string{}, hashes)
// Test setting hashes
provider.SetHashes(apiServerIDHash, configHash)
hashes = provider.GetCurrentHashes()
assertHashesEqual(t, []string{apiServerIDHash, configHash}, hashes)
// Test updating hashes
newConfigHash := "sha256:hash2"
provider.SetHashes(apiServerIDHash, newConfigHash)
hashes = provider.GetCurrentHashes()
assertHashesEqual(t, []string{apiServerIDHash, newConfigHash}, hashes)
// Test empty hashes
provider.SetHashes()
hashes = provider.GetCurrentHashes()
assertHashesEqual(t, []string{}, hashes)
}

View File

@@ -22,6 +22,7 @@ import (
"crypto/elliptic"
"crypto/rand"
"crypto/rsa"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/json"
@@ -1146,6 +1147,7 @@ jwt:
wantMetricStrings: []string{
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`,
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`,
},
},
{
@@ -1229,6 +1231,7 @@ jwt:
wantMetricStrings: []string{
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`,
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`,
},
},
{
@@ -1277,6 +1280,7 @@ jwt:
wantMetricStrings: []string{
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`,
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`,
},
},
{
@@ -1332,6 +1336,7 @@ jwt:
wantMetricStrings: []string{
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`,
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`,
},
},
{
@@ -1390,6 +1395,7 @@ jwt:
wantMetricStrings: []string{
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} FP`,
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} 1`,
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_old_config_hash"} 1`,
},
},
{
@@ -1433,6 +1439,7 @@ kind: AuthenticationConfiguration
wantMetricStrings: []string{
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 1`,
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_new_config_hash"} 1`,
},
},
{
@@ -1490,6 +1497,7 @@ jwt:
wantMetricStrings: []string{
`apiserver_authentication_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} FP`,
`apiserver_authentication_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="failure"} 1`,
`apiserver_authentication_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="replace_with_old_config_hash"} 1`,
},
},
}
@@ -1529,8 +1537,9 @@ jwt:
_ = tempFile.Close()
}()
newAuthConfig := tt.newAuthConfigFn(t, oidcServer.URL(), string(caCert))
// Write the new content to the temporary file
_, err = tempFile.Write([]byte(tt.newAuthConfigFn(t, oidcServer.URL(), string(caCert))))
_, err = tempFile.Write([]byte(newAuthConfig))
require.NoError(t, err)
// Atomically replace the original file with the temporary file
@@ -1565,6 +1574,16 @@ jwt:
_, err = client.CoreV1().Pods(defaultNamespace).List(ctx, metav1.ListOptions{})
tt.newAssertErrFn(t, err)
oldAuthConfigHash := getHash(tt.authConfigFn(t, oidcServer.URL(), string(caCert)))
newAuthConfigHash := getHash(newAuthConfig)
for i := range tt.wantMetricStrings {
if strings.Contains(tt.wantMetricStrings[i], "replace_with_new_config_hash") {
tt.wantMetricStrings[i] = strings.ReplaceAll(tt.wantMetricStrings[i], "replace_with_new_config_hash", newAuthConfigHash)
} else if strings.Contains(tt.wantMetricStrings[i], "replace_with_old_config_hash") {
tt.wantMetricStrings[i] = strings.ReplaceAll(tt.wantMetricStrings[i], "replace_with_old_config_hash", oldAuthConfigHash)
}
}
adminClient := kubernetes.NewForConfigOrDie(apiServer.ClientConfig)
body, err := adminClient.RESTClient().Get().AbsPath("/metrics").DoRaw(ctx)
require.NoError(t, err)
@@ -2114,3 +2133,10 @@ func testContext(t *testing.T) context.Context {
t.Cleanup(cancel)
return ctx
}
func getHash(data string) string {
if len(data) == 0 {
return ""
}
return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data)))
}

View File

@@ -735,8 +735,8 @@ authorizers:
if err != nil {
t.Fatal(err)
}
if initialMetrics.reloadSuccess == nil {
t.Fatal("expected success timestamp, got none")
if initialMetrics.reloadSuccess != nil {
t.Fatal("expected no success timestamp, got one")
}
if initialMetrics.reloadFailure != nil {
t.Fatal("expected no failure timestamp, got one")
@@ -754,19 +754,13 @@ authorizers:
if err != nil {
t.Fatal(err)
}
if reload1Metrics.reloadSuccess == nil {
t.Fatal("expected success timestamp, got none")
}
if !reload1Metrics.reloadSuccess.Equal(*initialMetrics.reloadSuccess) {
t.Fatalf("success timestamp changed from initial success %s to %s unexpectedly", initialMetrics.reloadSuccess.String(), reload1Metrics.reloadSuccess.String())
if reload1Metrics.reloadSuccess != nil {
t.Fatal("expected no success timestamp, got one")
}
if reload1Metrics.reloadFailure == nil {
t.Log("expected failure timestamp, got nil, retrying")
return false, nil
}
if !reload1Metrics.reloadFailure.After(*reload1Metrics.reloadSuccess) {
t.Fatalf("expected failure timestamp to be more recent than success timestamp, got %s <= %s", reload1Metrics.reloadFailure.String(), reload1Metrics.reloadSuccess.String())
}
return true, nil
})
if err != nil {
@@ -829,15 +823,19 @@ authorizers:
t.Fatalf("failure timestamp changed from reload1Metrics.reloadFailure %s to %s unexpectedly", reload1Metrics.reloadFailure.String(), reload2Metrics.reloadFailure.String())
}
if reload2Metrics.reloadSuccess == nil {
t.Fatal("expected success timestamp, got none")
}
if reload2Metrics.reloadSuccess.Equal(*initialMetrics.reloadSuccess) {
t.Log("success timestamp hasn't updated from initial success, retrying")
t.Log("expected success timestamp, got nil, retrying")
return false, nil
}
if !reload2Metrics.reloadSuccess.After(*reload2Metrics.reloadFailure) {
t.Fatalf("expected success timestamp to be more recent than failure, got %s <= %s", reload2Metrics.reloadSuccess.String(), reload2Metrics.reloadFailure.String())
}
if len(reload2Metrics.configHash) == 0 {
t.Fatal("expected config hash, got none")
}
if reload2Metrics.configHash == initialMetrics.configHash {
t.Logf("config hash %s is the same as initial %s, retrying", reload2Metrics.configHash, initialMetrics.configHash)
return false, nil
}
return true, nil
})
if err != nil {
@@ -883,6 +881,12 @@ authorizers:
if !reload3Metrics.reloadSuccess.Equal(*reload2Metrics.reloadSuccess) {
t.Fatalf("success timestamp changed from %s to %s unexpectedly", reload2Metrics.reloadSuccess.String(), reload3Metrics.reloadSuccess.String())
}
if len(reload3Metrics.configHash) == 0 {
t.Fatal("expected config hash, got none")
}
if reload3Metrics.configHash != reload2Metrics.configHash {
t.Fatalf("expected config hash to be the same as reload2Metrics %s, got %s", reload2Metrics.configHash, reload3Metrics.configHash)
}
if reload3Metrics.reloadFailure == nil {
t.Log("expected failure timestamp, got nil, retrying")
return false, nil
@@ -925,6 +929,7 @@ authorizers:
type metrics struct {
reloadSuccess *time.Time
reloadFailure *time.Time
configHash string
decisions map[authorizerKey]map[string]int
exclusions int
evalErrors int
@@ -945,12 +950,14 @@ var webhookMatchConditionEvalErrorMetric = regexp.MustCompile(`apiserver_authori
var whTotalMetric = regexp.MustCompile(`apiserver_authorization_webhook_evaluations_total{name="(.*?)",result="(.*?)"} (\d+)`)
var webhookDurationMetric = regexp.MustCompile(`apiserver_authorization_webhook_duration_seconds_count{name="(.*?)",result="(.*?)"} (\d+)`)
var webhookFailOpenMetric = regexp.MustCompile(`apiserver_authorization_webhook_evaluations_fail_open_total{name="(.*?)",result="(.*?)"} (\d+)`)
var configInfoMetric = regexp.MustCompile(`apiserver_authorization_config_controller_last_config_info\{apiserver_id_hash="sha256:[^"]*",hash="([^"]*)"\} (\d+)`)
func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) {
data, err := client.RESTClient().Get().AbsPath("/metrics").DoRaw(context.TODO())
// apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",status="success"} 1.7002567356895502e+09
// apiserver_authorization_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",status="failure"} 1.7002567356895502e+09
// apiserver_authorization_config_controller_automatic_reload_last_config_info{apiserver_id_hash="sha256:4b86cfa719a83dd63a4dc6a9831edb2b59240d0f59cf215b2d51aacb3f5c395e",hash="sha256:f309dd9c31fe24b3e594d2f9420419c48dfe954523245d5f35dc37739970d881"} 1
// apiserver_authorization_decisions_total{decision="allowed",name="allow.example.com",type="Webhook"} 2
// apiserver_authorization_decisions_total{decision="allowed",name="allowreloaded.example.com",type="Webhook"} 1
// apiserver_authorization_decisions_total{decision="denied",name="deny.example.com",type="Webhook"} 1
@@ -1052,6 +1059,12 @@ func getMetrics(t *testing.T, client *clientset.Clientset) (*metrics, error) {
t.Log("failure", m.reloadFailure.String())
}
}
if matches := configInfoMetric.FindStringSubmatch(line); matches != nil {
t.Logf("line: %s\n", line)
t.Log(matches)
m.configHash = matches[1]
t.Logf("config hash: %s", m.configHash)
}
}
return &m, nil
}

View File

@@ -23,6 +23,7 @@ import (
"bytes"
"context"
"crypto/aes"
"crypto/sha256"
"encoding/base64"
"encoding/binary"
"fmt"
@@ -352,13 +353,6 @@ resources:
t.Fatal(err)
}
// assert that the metrics we collect during the test run match expectations
// NOTE: 2 successful automatic reload resulted from 2 config file updates
wantMetricStrings := []string{
`apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
`apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 2`,
}
test.secret, err = test.createSecret(testSecret, testNamespace)
if err != nil {
t.Fatalf("Failed to create test secret, error: %v", err)
@@ -582,6 +576,15 @@ resources:
if err != nil {
t.Fatal(err)
}
// assert that the metrics we collect during the test run match expectations
// NOTE: 2 successful automatic reload resulted from 2 config file updates
wantMetricStrings := []string{
`apiserver_encryption_config_controller_automatic_reload_last_timestamp_seconds{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} FP`,
`apiserver_encryption_config_controller_automatic_reloads_total{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",status="success"} 2`,
fmt.Sprintf(`apiserver_encryption_config_controller_last_config_info{apiserver_id_hash="sha256:3c607df3b2bf22c9d9f01d5314b4bbf411c48ef43ff44ff29b1d55b41367c795",hash="%s"} 1`, getHash(encryptionConfigWithoutOldProvider)),
}
defer func() {
body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx)
if err != nil {
@@ -1205,3 +1208,10 @@ resources:
// the healthz check should be OK.
mustBeHealthy(t, "/kms-providers", "ok", test.kubeAPIServer.ClientConfig)
}
func getHash(data string) string {
if len(data) == 0 {
return ""
}
return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data)))
}